diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py index ffc9402232..584f476fc0 100644 --- a/pylint/checkers/__init__.py +++ b/pylint/checkers/__init__.py @@ -46,28 +46,35 @@ """ +from typing import Iterable, List, Union + from pylint.checkers.base_checker import BaseChecker, BaseTokenChecker from pylint.checkers.deprecated import DeprecatedMixin from pylint.checkers.mapreduce_checker import MapReduceMixin +from pylint.typing import CheckerStats from pylint.utils import diff_string, register_plugins -def table_lines_from_stats(stats, old_stats, columns): +def table_lines_from_stats( + stats: CheckerStats, + old_stats: CheckerStats, + columns: Iterable[str], +) -> List[str]: """get values listed in from and , and return a formated list of values, designed to be given to a ureport.Table object """ - lines = [] + lines: List[str] = [] for m_type in columns: - new = stats[m_type] - old = old_stats.get(m_type) + new: Union[int, str] = stats[m_type] # type: ignore + old: Union[int, str, None] = old_stats.get(m_type) # type: ignore if old is not None: diff_str = diff_string(old, new) else: old, diff_str = "NC", "NC" new = f"{new:.3f}" if isinstance(new, float) else str(new) old = f"{old:.3f}" if isinstance(old, float) else str(old) - lines += (m_type.replace("_", " "), new, old, diff_str) + lines.extend((m_type.replace("_", " "), new, old, diff_str)) return lines diff --git a/pylint/checkers/base.py b/pylint/checkers/base.py index d839d921e1..2f1579cd6a 100644 --- a/pylint/checkers/base.py +++ b/pylint/checkers/base.py @@ -66,7 +66,7 @@ import itertools import re import sys -from typing import Any, Iterator, Optional, Pattern +from typing import Any, Dict, Iterator, Optional, Pattern, Union import astroid from astroid import nodes @@ -81,6 +81,7 @@ is_property_setter, ) from pylint.reporters.ureports import nodes as reporter_nodes +from pylint.typing import CheckerStats class NamingStyle: @@ -386,36 +387,42 @@ def _has_abstract_methods(node): return len(utils.unimplemented_abstract_methods(node)) > 0 -def report_by_type_stats(sect, stats, old_stats): +def report_by_type_stats( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """make a report of * percentage of different types documented * percentage of different types with a bad name """ # percentage of different types documented and/or with a bad name - nice_stats = {} + nice_stats: Dict[str, Dict[str, str]] = {} for node_type in ("module", "class", "method", "function"): try: - total = stats[node_type] + total: int = stats[node_type] # type: ignore except KeyError as e: raise exceptions.EmptyReportError() from e nice_stats[node_type] = {} if total != 0: try: - documented = total - stats["undocumented_" + node_type] + undocumented_node: int = stats["undocumented_" + node_type] # type: ignore + documented = total - undocumented_node percent = (documented * 100.0) / total nice_stats[node_type]["percent_documented"] = f"{percent:.2f}" except KeyError: nice_stats[node_type]["percent_documented"] = "NC" try: - percent = (stats["badname_" + node_type] * 100.0) / total + badname_node: int = stats["badname_" + node_type] # type: ignore + percent = (badname_node * 100.0) / total nice_stats[node_type]["percent_badname"] = f"{percent:.2f}" except KeyError: nice_stats[node_type]["percent_badname"] = "NC" lines = ["type", "number", "old number", "difference", "%documented", "%badname"] for node_type in ("module", "class", "method", "function"): new = stats[node_type] - old = old_stats.get(node_type, None) + old: Optional[Union[str, int]] = old_stats.get(node_type, None) # type: ignore if old is not None: diff_str = lint_utils.diff_string(old, new) else: @@ -1082,7 +1089,7 @@ class BasicChecker(_BasicChecker): def __init__(self, linter): _BasicChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} self._tryfinallys = None def open(self): @@ -1159,13 +1166,13 @@ def _check_using_constant_test(self, node, test): def visit_module(self, _: nodes.Module) -> None: """check module name, docstring and required arguments""" - self.stats["module"] += 1 + self.stats["module"] += 1 # type: ignore def visit_classdef(self, _: nodes.ClassDef) -> None: """check module name, docstring and redefinition increment branch counter """ - self.stats["class"] += 1 + self.stats["class"] += 1 # type: ignore @utils.check_messages( "pointless-statement", "pointless-string-statement", "expression-not-assigned" @@ -1304,7 +1311,7 @@ def visit_functiondef(self, node: nodes.FunctionDef) -> None: """check function name, docstring, arguments, redefinition, variable names, max locals """ - self.stats["method" if node.is_method() else "function"] += 1 + self.stats["method" if node.is_method() else "function"] += 1 # type: ignore self._check_dangerous_default(node) visit_asyncfunctiondef = visit_functiondef @@ -2040,7 +2047,7 @@ def _raise_name_warning( ) self.add_message(warning, node=node, args=args, confidence=confidence) - self.stats["badname_" + node_type] += 1 + self.stats["badname_" + node_type] += 1 # type: ignore def _name_allowed_by_regex(self, name: str) -> bool: return name in self.config.good_names or any( diff --git a/pylint/checkers/base_checker.py b/pylint/checkers/base_checker.py index 5dfd7ea782..791a3cb1e0 100644 --- a/pylint/checkers/base_checker.py +++ b/pylint/checkers/base_checker.py @@ -24,6 +24,7 @@ from pylint.exceptions import InvalidMessageError from pylint.interfaces import UNDEFINED, IRawChecker, ITokenChecker, implements from pylint.message.message_definition import MessageDefinition +from pylint.typing import CheckerStats from pylint.utils import get_rst_section, get_rst_title @@ -51,6 +52,7 @@ def __init__(self, linter=None): self.name = self.name.lower() OptionsProviderMixIn.__init__(self) self.linter = linter + self.stats: CheckerStats = {} def __gt__(self, other): """Permit to sort a list of Checker by name.""" diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index 9777b568e6..362d299de1 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -35,6 +35,7 @@ from pylint.checkers import BaseChecker from pylint.checkers.utils import check_messages from pylint.interfaces import IAstroidChecker +from pylint.typing import CheckerStats MSGS = { # pylint: disable=consider-using-namedtuple-or-dataclass "R0901": ( @@ -391,7 +392,7 @@ class MisdesignChecker(BaseChecker): def __init__(self, linter=None): BaseChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} self._returns = None self._branches = None self._stmts = None diff --git a/pylint/checkers/imports.py b/pylint/checkers/imports.py index 7e0cda26dc..8b413253fc 100644 --- a/pylint/checkers/imports.py +++ b/pylint/checkers/imports.py @@ -68,6 +68,7 @@ from pylint.interfaces import IAstroidChecker from pylint.lint import PyLinter from pylint.reporters.ureports.nodes import Paragraph, VerbatimText, VNode +from pylint.typing import CheckerStats from pylint.utils import IsortDriver, get_global_option @@ -423,7 +424,7 @@ def __init__( self, linter: PyLinter = None ): # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941 BaseChecker.__init__(self, linter) - self.stats: Dict[Any, Any] = {} + self.stats: CheckerStats = {} self.import_graph: collections.defaultdict = collections.defaultdict(set) self._imports_stack: List[Tuple[Any, Any]] = [] self._first_non_import_node = None @@ -839,9 +840,8 @@ def _add_imported_module( self._module_pkg[context_name] = context_name.rsplit(".", 1)[0] # handle dependencies - importedmodnames = self.stats["dependencies"].setdefault( - importedmodname, set() - ) + dependencies_stat: Dict[str, Union[Set]] = self.stats["dependencies"] # type: ignore + importedmodnames = dependencies_stat.setdefault(importedmodname, set()) if context_name not in importedmodnames: importedmodnames.add(context_name) diff --git a/pylint/checkers/raw_metrics.py b/pylint/checkers/raw_metrics.py index 028c68e7ab..cc3f2729d6 100644 --- a/pylint/checkers/raw_metrics.py +++ b/pylint/checkers/raw_metrics.py @@ -15,27 +15,32 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import tokenize -from typing import Any +from typing import Any, Optional, Union from pylint.checkers import BaseTokenChecker from pylint.exceptions import EmptyReportError from pylint.interfaces import ITokenChecker from pylint.reporters.ureports.nodes import Table +from pylint.typing import CheckerStats from pylint.utils import diff_string -def report_raw_stats(sect, stats, old_stats): +def report_raw_stats( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """calculate percentage of code / doc / comment / empty""" - total_lines = stats["total_lines"] + total_lines: int = stats["total_lines"] # type: ignore if not total_lines: raise EmptyReportError() sect.description = f"{total_lines} lines have been analyzed" lines = ["type", "number", "%", "previous", "difference"] for node_type in ("code", "docstring", "comment", "empty"): key = node_type + "_lines" - total = stats[key] + total: int = stats[key] # type: ignore percent = float(total * 100) / total_lines - old = old_stats.get(key, None) + old: Optional[Union[int, str]] = old_stats.get(key, None) # type: ignore if old is not None: diff_str = diff_string(old, total) else: @@ -66,7 +71,7 @@ class RawMetricsChecker(BaseTokenChecker): def __init__(self, linter): BaseTokenChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} def open(self): """init statistics""" diff --git a/pylint/checkers/similar.py b/pylint/checkers/similar.py index 4ceb5d6603..2a28e7d6c5 100644 --- a/pylint/checkers/similar.py +++ b/pylint/checkers/similar.py @@ -73,6 +73,7 @@ from pylint.checkers import BaseChecker, MapReduceMixin, table_lines_from_stats from pylint.interfaces import IRawChecker from pylint.reporters.ureports.nodes import Table +from pylint.typing import CheckerStats from pylint.utils import decoding_stream DEFAULT_MIN_SIMILARITY_LINE = 4 @@ -721,7 +722,11 @@ def real_lines(self): } -def report_similarities(sect, stats, old_stats): +def report_similarities( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """make a layout with some stats about duplication""" lines = ["", "now", "previous", "difference"] lines += table_lines_from_stats( @@ -804,7 +809,7 @@ def __init__(self, linter=None) -> None: ignore_imports=self.config.ignore_imports, ignore_signatures=self.config.ignore_signatures, ) - self.stats = None + self.stats: CheckerStats = {} def set_option(self, optname, value, action=None, optdict=None): """method called to set an option (registered in the options list) diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index ad6830721d..2f3d0dd050 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -3,10 +3,15 @@ import collections import functools +from typing import TYPE_CHECKING, Dict, List, Union from pylint import reporters from pylint.lint.utils import _patch_sys_path from pylint.message import Message +from pylint.typing import CheckerStats + +if TYPE_CHECKING: + from typing import Counter # typing.Counter added in Python 3.6.1 try: import multiprocessing @@ -30,20 +35,20 @@ def _get_new_args(message): return (message.msg_id, message.symbol, location, message.msg, message.confidence) -def _merge_stats(stats): - merged = {} - by_msg = collections.Counter() +def _merge_stats(stats: List[CheckerStats]): + merged: CheckerStats = {} + by_msg: "Counter[str]" = collections.Counter() for stat in stats: - message_stats = stat.pop("by_msg", {}) + message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore by_msg.update(message_stats) for key, item in stat.items(): if key not in merged: merged[key] = item elif isinstance(item, dict): - merged[key].update(item) + merged[key].update(item) # type: ignore else: - merged[key] = merged[key] + item + merged[key] = merged[key] + item # type: ignore merged["by_msg"] = by_msg return merged diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 54614ad890..d731e71b3f 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -31,6 +31,7 @@ ) from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn from pylint.reporters.ureports import nodes as report_nodes +from pylint.typing import CheckerStats from pylint.utils import ASTWalker, FileState, utils from pylint.utils.pragma_parser import ( OPTION_PO, @@ -502,7 +503,7 @@ def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None): self.file_state = FileState() self.current_name = None self.current_file = None - self.stats = None + self.stats: CheckerStats = {} self.fail_on_symbols = [] # init options self._external_opts = options @@ -729,8 +730,10 @@ def enable_fail_on_messages(self): self.fail_on_symbols.append(msg.symbol) def any_fail_on_issues(self): - return self.stats is not None and any( - x in self.fail_on_symbols for x in self.stats["by_msg"] + return ( + self.stats + and self.stats.get("by_msg") is not None + and any(x in self.fail_on_symbols for x in self.stats["by_msg"]) ) def disable_noerror_messages(self): diff --git a/pylint/lint/report_functions.py b/pylint/lint/report_functions.py index 21cb3b8245..12c4b03352 100644 --- a/pylint/lint/report_functions.py +++ b/pylint/lint/report_functions.py @@ -2,12 +2,18 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections +from typing import DefaultDict, Dict, List, Tuple, Union from pylint import checkers, exceptions from pylint.reporters.ureports import nodes as report_nodes +from pylint.typing import CheckerStats -def report_total_messages_stats(sect, stats, previous_stats): +def report_total_messages_stats( + sect, + stats: CheckerStats, + previous_stats: CheckerStats, +): """make total errors / warnings report""" lines = ["type", "number", "previous", "difference"] lines += checkers.table_lines_from_stats( @@ -16,14 +22,19 @@ def report_total_messages_stats(sect, stats, previous_stats): sect.append(report_nodes.Table(children=lines, cols=4, rheaders=1)) -def report_messages_stats(sect, stats, _): +def report_messages_stats( + sect, + stats: CheckerStats, + _: CheckerStats, +): """make messages type report""" if not stats["by_msg"]: # don't print this report when we didn't detected any errors raise exceptions.EmptyReportError() - in_order = sorted( + by_msg_stats: Dict[str, int] = stats["by_msg"] # type: ignore + in_order: List[Tuple[int, str]] = sorted( (value, msg_id) - for msg_id, value in stats["by_msg"].items() + for msg_id, value in by_msg_stats.items() if not msg_id.startswith("I") ) in_order.reverse() @@ -33,16 +44,23 @@ def report_messages_stats(sect, stats, _): sect.append(report_nodes.Table(children=lines, cols=2, rheaders=1)) -def report_messages_by_module_stats(sect, stats, _): +def report_messages_by_module_stats( + sect, + stats: CheckerStats, + _: CheckerStats, +): """make errors / warnings by modules report""" - if len(stats["by_module"]) == 1: + module_stats: Dict[str, Dict[str, int]] = stats["by_module"] # type: ignore + if len(module_stats) == 1: # don't print this report when we are analysing a single module raise exceptions.EmptyReportError() - by_mod = collections.defaultdict(dict) + by_mod: DefaultDict[str, Dict[str, Union[int, float]]] = collections.defaultdict( + dict + ) for m_type in ("fatal", "error", "warning", "refactor", "convention"): - total = stats[m_type] - for module in stats["by_module"].keys(): - mod_total = stats["by_module"][module][m_type] + total: int = stats[m_type] # type: ignore + for module in module_stats.keys(): + mod_total = module_stats[module][m_type] percent = 0 if total == 0 else float((mod_total) * 100) / total by_mod[module][m_type] = percent sorted_result = [] diff --git a/pylint/reporters/base_reporter.py b/pylint/reporters/base_reporter.py index fda99f4ef9..4cf5d5895d 100644 --- a/pylint/reporters/base_reporter.py +++ b/pylint/reporters/base_reporter.py @@ -6,6 +6,7 @@ from typing import List from pylint.message import Message +from pylint.typing import CheckerStats class BaseReporter: @@ -41,7 +42,7 @@ def writeln(self, string=""): def display_reports(self, layout): """display results encapsulated in the layout tree""" self.section = 0 - if hasattr(layout, "report_id"): + if layout.report_id: layout.children[0].children[0].data += f" ({layout.report_id})" self._display(layout) @@ -65,5 +66,9 @@ def display_messages(self, layout): def on_set_current_module(self, module, filepath): """Hook called when a module starts to be analysed.""" - def on_close(self, stats, previous_stats): + def on_close( + self, + stats: CheckerStats, + previous_stats: CheckerStats, + ) -> None: """Hook called when a module finished analyzing.""" diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py index a4dbae53b7..245c10f79c 100644 --- a/pylint/reporters/multi_reporter.py +++ b/pylint/reporters/multi_reporter.py @@ -3,12 +3,13 @@ import os -from typing import IO, Any, AnyStr, Callable, List, Mapping, Optional, Union +from typing import IO, Any, AnyStr, Callable, List, Optional, Union from pylint.interfaces import IReporter from pylint.message import Message from pylint.reporters.base_reporter import BaseReporter from pylint.reporters.ureports.nodes import BaseLayout +from pylint.typing import CheckerStats AnyFile = IO[AnyStr] AnyPath = Union[str, bytes, os.PathLike] @@ -95,8 +96,8 @@ def on_set_current_module(self, module: str, filepath: Optional[AnyPath]) -> Non def on_close( self, - stats: Mapping[Any, Any], - previous_stats: Mapping[Any, Any], + stats: CheckerStats, + previous_stats: CheckerStats, ) -> None: """hook called when a module finished analyzing""" for rep in self._sub_reporters: diff --git a/pylint/reporters/reports_handler_mix_in.py b/pylint/reporters/reports_handler_mix_in.py index 914556ef48..450d383f51 100644 --- a/pylint/reporters/reports_handler_mix_in.py +++ b/pylint/reporters/reports_handler_mix_in.py @@ -2,9 +2,14 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections +from typing import TYPE_CHECKING, Any from pylint.exceptions import EmptyReportError from pylint.reporters.ureports.nodes import Section +from pylint.typing import CheckerStats + +if TYPE_CHECKING: + from pylint.lint.pylinter import PyLinter class ReportsHandlerMixIn: @@ -49,7 +54,11 @@ def report_is_enabled(self, reportid): """ return self._reports_state.get(reportid, True) - def make_reports(self, stats, old_stats): + def make_reports( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter + self: "PyLinter", + stats: CheckerStats, + old_stats: CheckerStats, + ): """render registered reports""" sect = Section("Report", f"{self.stats['statement']} statements analysed.") for checker in self.report_order(): @@ -65,7 +74,9 @@ def make_reports(self, stats, old_stats): sect.append(report_sect) return sect - def add_stats(self, **kwargs): + def add_stats( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter + self: "PyLinter", **kwargs: Any + ) -> CheckerStats: """add some stats entries to the statistic dictionary raise an AssertionError if there is a key conflict """ diff --git a/pylint/reporters/ureports/nodes.py b/pylint/reporters/ureports/nodes.py index 3f7842a18e..c5c14737da 100644 --- a/pylint/reporters/ureports/nodes.py +++ b/pylint/reporters/ureports/nodes.py @@ -14,6 +14,7 @@ A micro report is a tree of layout and content objects. """ +from typing import Optional class VNode: @@ -126,6 +127,7 @@ def __init__(self, title=None, description=None, **kwargs): self.insert(0, Paragraph([Text(description)])) if title: self.insert(0, Title(children=(title,))) + self.report_id: Optional[str] = None class EvaluationSection(Section): diff --git a/pylint/testutils/unittest_linter.py b/pylint/testutils/unittest_linter.py index 2d0a4f2c3f..b75592ce6a 100644 --- a/pylint/testutils/unittest_linter.py +++ b/pylint/testutils/unittest_linter.py @@ -3,6 +3,7 @@ from pylint.testutils.global_test_linter import linter from pylint.testutils.output_line import Message +from pylint.typing import CheckerStats class UnittestLinter: @@ -12,7 +13,7 @@ class UnittestLinter: def __init__(self): self._messages = [] - self.stats = {} + self.stats: CheckerStats = {} def release_messages(self): try: diff --git a/pylint/typing.py b/pylint/typing.py new file mode 100644 index 0000000000..9ce1c587f7 --- /dev/null +++ b/pylint/typing.py @@ -0,0 +1,14 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +"""A collection of typing utilities.""" +from typing import TYPE_CHECKING, Dict, List, Union + +if TYPE_CHECKING: + from typing import Counter # typing.Counter added in Python 3.6.1 + + +# The base type of the "stats" attribute of a checker +CheckerStats = Dict[ + str, Union[int, "Counter[str]", List, Dict[str, Union[int, str, Dict[str, int]]]] +] diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py index 18656ad740..c31cbdba06 100644 --- a/tests/lint/unittest_lint.py +++ b/tests/lint/unittest_lint.py @@ -48,7 +48,7 @@ from os import chdir, getcwd from os.path import abspath, basename, dirname, isdir, join, sep from shutil import rmtree -from typing import Iterable, Iterator, List, Optional, Tuple +from typing import Dict, Iterable, Iterator, List, Optional, Tuple import platformdirs import pytest @@ -828,7 +828,8 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None: linter = init_linter linter.check(os.path.join(os.path.dirname(__file__), "data")) - for module, module_stats in linter.stats["by_module"].items(): + by_module_stats: Dict[str, Dict[str, int]] = linter.stats["by_module"] # type: ignore + for module, module_stats in by_module_stats.items(): linter2 = init_linter if module == "data": diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py index 102c0f1225..66236ac8dc 100644 --- a/tests/test_check_parallel.py +++ b/tests/test_check_parallel.py @@ -24,6 +24,7 @@ from pylint.lint.parallel import _worker_initialize as worker_initialize from pylint.lint.parallel import check_parallel from pylint.testutils import GenericTestReporter as Reporter +from pylint.typing import CheckerStats def _gen_file_data(idx: int = 0) -> Tuple[str, str, str]: @@ -101,7 +102,7 @@ def __init__(self, linter: PyLinter) -> None: super().__init__(linter) self.data: List[str] = [] self.linter = linter - self.stats = None + self.stats: CheckerStats = {} def open(self) -> None: """init the checkers: reset statistics information""" diff --git a/tests/test_regr.py b/tests/test_regr.py index 793a196129..12dd03406a 100644 --- a/tests/test_regr.py +++ b/tests/test_regr.py @@ -23,7 +23,7 @@ import os import sys from os.path import abspath, dirname, join -from typing import Iterator +from typing import Dict, Iterator import astroid import pytest @@ -111,12 +111,14 @@ def modify_path() -> Iterator: def test_check_package___init__(finalize_linter: PyLinter) -> None: filename = "package.__init__" finalize_linter.check(filename) - checked = list(finalize_linter.stats["by_module"].keys()) + by_module_stats: Dict[str, Dict[str, int]] = finalize_linter.stats["by_module"] # type: ignore + checked = list(by_module_stats.keys()) assert checked == [filename] os.chdir(join(REGR_DATA, "package")) finalize_linter.check("__init__") - checked = list(finalize_linter.stats["by_module"].keys()) + by_module_stats: Dict[str, Dict[str, int]] = finalize_linter.stats["by_module"] # type: ignore + checked = list(by_module_stats.keys()) assert checked == ["__init__"]