From 0687d5caf7a57ebf9c82126e92a3c223d724f5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Tue, 14 Sep 2021 23:03:02 +0200 Subject: [PATCH 1/4] Change lines to lists (#5009) --- pylint/checkers/base.py | 6 +++--- pylint/checkers/raw_metrics.py | 4 ++-- pylint/lint/report_functions.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pylint/checkers/base.py b/pylint/checkers/base.py index 2ca79453a7..d839d921e1 100644 --- a/pylint/checkers/base.py +++ b/pylint/checkers/base.py @@ -412,7 +412,7 @@ def report_by_type_stats(sect, stats, old_stats): nice_stats[node_type]["percent_badname"] = f"{percent:.2f}" except KeyError: nice_stats[node_type]["percent_badname"] = "NC" - lines = ("type", "number", "old number", "difference", "%documented", "%badname") + lines = ["type", "number", "old number", "difference", "%documented", "%badname"] for node_type in ("module", "class", "method", "function"): new = stats[node_type] old = old_stats.get(node_type, None) @@ -420,14 +420,14 @@ def report_by_type_stats(sect, stats, old_stats): diff_str = lint_utils.diff_string(old, new) else: old, diff_str = "NC", "NC" - lines += ( + lines += [ node_type, str(new), str(old), diff_str, nice_stats[node_type].get("percent_documented", "0"), nice_stats[node_type].get("percent_badname", "0"), - ) + ] sect.append(reporter_nodes.Table(children=lines, cols=6, rheaders=1)) diff --git a/pylint/checkers/raw_metrics.py b/pylint/checkers/raw_metrics.py index eb3f717ff1..028c68e7ab 100644 --- a/pylint/checkers/raw_metrics.py +++ b/pylint/checkers/raw_metrics.py @@ -30,7 +30,7 @@ def report_raw_stats(sect, stats, old_stats): if not total_lines: raise EmptyReportError() sect.description = f"{total_lines} lines have been analyzed" - lines = ("type", "number", "%", "previous", "difference") + lines = ["type", "number", "%", "previous", "difference"] for node_type in ("code", "docstring", "comment", "empty"): key = node_type + "_lines" total = stats[key] @@ -40,7 +40,7 @@ def report_raw_stats(sect, stats, old_stats): diff_str = diff_string(old, total) else: old, diff_str = "NC", "NC" - lines += (node_type, str(total), f"{percent:.2f}", str(old), diff_str) + lines += [node_type, str(total), f"{percent:.2f}", str(old), diff_str] sect.append(Table(children=lines, cols=5, rheaders=1)) diff --git a/pylint/lint/report_functions.py b/pylint/lint/report_functions.py index fd316c611f..21cb3b8245 100644 --- a/pylint/lint/report_functions.py +++ b/pylint/lint/report_functions.py @@ -27,9 +27,9 @@ def report_messages_stats(sect, stats, _): if not msg_id.startswith("I") ) in_order.reverse() - lines = ("message id", "occurrences") + lines = ["message id", "occurrences"] for value, msg_id in in_order: - lines += (msg_id, str(value)) + lines += [msg_id, str(value)] sect.append(report_nodes.Table(children=lines, cols=2, rheaders=1)) From a3f3405d57558d6d5dd21112a5dfeb1fb1ed50a5 Mon Sep 17 00:00:00 2001 From: Pierre Sassoulas Date: Wed, 15 Sep 2021 07:43:05 +0200 Subject: [PATCH 2/4] Fix PyreverseConfig imports in pyreverse's tests --- setup.cfg | 3 +++ tests/pyreverse/test_diadefs.py | 2 +- tests/pyreverse/test_writer.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7e18ba310c..f415db45d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -89,6 +89,9 @@ warn_unused_ignores = True [mypy-astroid.*] ignore_missing_imports = True +[mypy-tests.*] +ignore_missing_imports = True + [mypy-coverage] ignore_missing_imports = True diff --git a/tests/pyreverse/test_diadefs.py b/tests/pyreverse/test_diadefs.py index 3c2ab7fda9..622ad59eb1 100644 --- a/tests/pyreverse/test_diadefs.py +++ b/tests/pyreverse/test_diadefs.py @@ -26,7 +26,7 @@ import pytest from astroid import nodes -from conftest import PyreverseConfig # type: ignore #pylint: disable=no-name-in-module +from tests.pyreverse.conftest import PyreverseConfig from pylint.pyreverse.diadefslib import ( ClassDiadefGenerator, diff --git a/tests/pyreverse/test_writer.py b/tests/pyreverse/test_writer.py index 9c847be299..e0bcfaaa40 100644 --- a/tests/pyreverse/test_writer.py +++ b/tests/pyreverse/test_writer.py @@ -28,7 +28,7 @@ from unittest.mock import Mock import pytest -from conftest import PyreverseConfig # type: ignore #pylint: disable=no-name-in-module +from tests.pyreverse.conftest import PyreverseConfig from pylint.pyreverse.diadefslib import DefaultDiadefGenerator, DiadefsHandler from pylint.pyreverse.inspector import Linker, Project From cb896128b0e8f62c0650e980ef77a3c8af21ef8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Wed, 15 Sep 2021 18:30:46 +0200 Subject: [PATCH 3/4] Remove most `typing.cast()` calls (#4995) --- pylint/checkers/classes.py | 6 +----- pylint/checkers/refactoring/recommendation_checker.py | 10 +--------- pylint/checkers/refactoring/refactoring_checker.py | 4 +--- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pylint/checkers/classes.py b/pylint/checkers/classes.py index 4e208f0649..b02508424c 100644 --- a/pylint/checkers/classes.py +++ b/pylint/checkers/classes.py @@ -47,7 +47,7 @@ """ import collections from itertools import chain, zip_longest -from typing import List, Pattern, cast +from typing import List, Pattern import astroid from astroid import nodes @@ -907,7 +907,6 @@ def leave_classdef(self, node: nodes.ClassDef) -> None: def _check_unused_private_functions(self, node: nodes.ClassDef) -> None: for function_def in node.nodes_of_class(nodes.FunctionDef): - function_def = cast(nodes.FunctionDef, function_def) if not is_attr_private(function_def.name): continue parent_scope = function_def.parent.scope() @@ -918,7 +917,6 @@ def _check_unused_private_functions(self, node: nodes.ClassDef) -> None: ): continue for attribute in node.nodes_of_class(nodes.Attribute): - attribute = cast(nodes.Attribute, attribute) if ( attribute.attrname != function_def.name or attribute.scope() == function_def # We ignore recursive calls @@ -978,7 +976,6 @@ def _check_unused_private_variables(self, node: nodes.ClassDef) -> None: def _check_unused_private_attributes(self, node: nodes.ClassDef) -> None: for assign_attr in node.nodes_of_class(nodes.AssignAttr): - assign_attr = cast(nodes.AssignAttr, assign_attr) if not is_attr_private(assign_attr.attrname) or not isinstance( assign_attr.expr, nodes.Name ): @@ -999,7 +996,6 @@ def _check_unused_private_attributes(self, node: nodes.ClassDef) -> None: ) for attribute in node.nodes_of_class(nodes.Attribute): - attribute = cast(nodes.Attribute, attribute) if attribute.attrname != assign_attr.attrname: continue diff --git a/pylint/checkers/refactoring/recommendation_checker.py b/pylint/checkers/refactoring/recommendation_checker.py index a765eca013..84c3999885 100644 --- a/pylint/checkers/refactoring/recommendation_checker.py +++ b/pylint/checkers/refactoring/recommendation_checker.py @@ -1,6 +1,6 @@ # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE -from typing import Union, cast +from typing import Union import astroid from astroid import nodes @@ -128,17 +128,14 @@ def _check_use_maxsplit_arg(self, node: nodes.Call) -> None: # Check if loop present within the scope of the node scope = node.scope() for loop_node in scope.nodes_of_class((nodes.For, nodes.While)): - loop_node = cast(nodes.NodeNG, loop_node) if not loop_node.parent_of(node): continue # Check if var is mutated within loop (Assign/AugAssign) for assignment_node in loop_node.nodes_of_class(nodes.AugAssign): - assignment_node = cast(nodes.AugAssign, assignment_node) if node.parent.slice.name == assignment_node.target.name: return for assignment_node in loop_node.nodes_of_class(nodes.Assign): - assignment_node = cast(nodes.Assign, assignment_node) if node.parent.slice.name in [ n.name for n in assignment_node.targets ]: @@ -216,7 +213,6 @@ def _check_consider_using_enumerate(self, node: nodes.For) -> None: # for body. for child in node.body: for subscript in child.nodes_of_class(nodes.Subscript): - subscript = cast(nodes.Subscript, subscript) if not isinstance(subscript.value, expected_subscript_val_type): continue @@ -254,8 +250,6 @@ def _check_consider_using_dict_items(self, node: nodes.For) -> None: # for body. for child in node.body: for subscript in child.nodes_of_class(nodes.Subscript): - subscript = cast(nodes.Subscript, subscript) - if not isinstance(subscript.value, (nodes.Name, nodes.Attribute)): continue @@ -304,8 +298,6 @@ def _check_consider_using_dict_items_comprehension( for child in node.parent.get_children(): for subscript in child.nodes_of_class(nodes.Subscript): - subscript = cast(nodes.Subscript, subscript) - if not isinstance(subscript.value, (nodes.Name, nodes.Attribute)): continue diff --git a/pylint/checkers/refactoring/refactoring_checker.py b/pylint/checkers/refactoring/refactoring_checker.py index 525e77112e..763c65d46c 100644 --- a/pylint/checkers/refactoring/refactoring_checker.py +++ b/pylint/checkers/refactoring/refactoring_checker.py @@ -6,7 +6,7 @@ import itertools import tokenize from functools import reduce -from typing import Dict, Iterator, List, NamedTuple, Optional, Tuple, Union, cast +from typing import Dict, Iterator, List, NamedTuple, Optional, Tuple, Union import astroid from astroid import nodes @@ -1862,8 +1862,6 @@ def _check_unnecessary_dict_index_lookup( ) for child in children: for subscript in child.nodes_of_class(nodes.Subscript): - subscript = cast(nodes.Subscript, subscript) - if not isinstance(subscript.value, (nodes.Name, nodes.Attribute)): continue From 22e56c07cf745d695df1d52fe3988cc071f0951b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Wed, 15 Sep 2021 20:42:22 +0200 Subject: [PATCH 4/4] Add typing to all calls to ``self.stats`` (#4973) * Add typing to all calls to ``self.stats`` All checkers inherit from a baseclass which has a ``stats`` attribute. This attribute has a fairly unmanageable type, but the current typing includes all variations of the attribute. Other changes not directly related to ``self.stats`` are due to ``mypy``warnings. This incorporate the feedback received in #4954 * Add ``CheckerStatistic`` class to ``pylint/typing`` * Guard `typing.Counter` import Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- pylint/checkers/__init__.py | 17 +++++++--- pylint/checkers/base.py | 31 +++++++++++------- pylint/checkers/base_checker.py | 2 ++ pylint/checkers/design_analysis.py | 3 +- pylint/checkers/imports.py | 8 ++--- pylint/checkers/raw_metrics.py | 17 ++++++---- pylint/checkers/similar.py | 9 +++-- pylint/lint/parallel.py | 17 ++++++---- pylint/lint/pylinter.py | 9 +++-- pylint/lint/report_functions.py | 38 ++++++++++++++++------ pylint/reporters/base_reporter.py | 9 +++-- pylint/reporters/multi_reporter.py | 7 ++-- pylint/reporters/reports_handler_mix_in.py | 15 +++++++-- pylint/reporters/ureports/nodes.py | 2 ++ pylint/testutils/unittest_linter.py | 3 +- pylint/typing.py | 14 ++++++++ tests/lint/unittest_lint.py | 5 +-- tests/test_check_parallel.py | 3 +- tests/test_regr.py | 8 +++-- 19 files changed, 154 insertions(+), 63 deletions(-) create mode 100644 pylint/typing.py diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py index ffc9402232..584f476fc0 100644 --- a/pylint/checkers/__init__.py +++ b/pylint/checkers/__init__.py @@ -46,28 +46,35 @@ """ +from typing import Iterable, List, Union + from pylint.checkers.base_checker import BaseChecker, BaseTokenChecker from pylint.checkers.deprecated import DeprecatedMixin from pylint.checkers.mapreduce_checker import MapReduceMixin +from pylint.typing import CheckerStats from pylint.utils import diff_string, register_plugins -def table_lines_from_stats(stats, old_stats, columns): +def table_lines_from_stats( + stats: CheckerStats, + old_stats: CheckerStats, + columns: Iterable[str], +) -> List[str]: """get values listed in from and , and return a formated list of values, designed to be given to a ureport.Table object """ - lines = [] + lines: List[str] = [] for m_type in columns: - new = stats[m_type] - old = old_stats.get(m_type) + new: Union[int, str] = stats[m_type] # type: ignore + old: Union[int, str, None] = old_stats.get(m_type) # type: ignore if old is not None: diff_str = diff_string(old, new) else: old, diff_str = "NC", "NC" new = f"{new:.3f}" if isinstance(new, float) else str(new) old = f"{old:.3f}" if isinstance(old, float) else str(old) - lines += (m_type.replace("_", " "), new, old, diff_str) + lines.extend((m_type.replace("_", " "), new, old, diff_str)) return lines diff --git a/pylint/checkers/base.py b/pylint/checkers/base.py index d839d921e1..2f1579cd6a 100644 --- a/pylint/checkers/base.py +++ b/pylint/checkers/base.py @@ -66,7 +66,7 @@ import itertools import re import sys -from typing import Any, Iterator, Optional, Pattern +from typing import Any, Dict, Iterator, Optional, Pattern, Union import astroid from astroid import nodes @@ -81,6 +81,7 @@ is_property_setter, ) from pylint.reporters.ureports import nodes as reporter_nodes +from pylint.typing import CheckerStats class NamingStyle: @@ -386,36 +387,42 @@ def _has_abstract_methods(node): return len(utils.unimplemented_abstract_methods(node)) > 0 -def report_by_type_stats(sect, stats, old_stats): +def report_by_type_stats( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """make a report of * percentage of different types documented * percentage of different types with a bad name """ # percentage of different types documented and/or with a bad name - nice_stats = {} + nice_stats: Dict[str, Dict[str, str]] = {} for node_type in ("module", "class", "method", "function"): try: - total = stats[node_type] + total: int = stats[node_type] # type: ignore except KeyError as e: raise exceptions.EmptyReportError() from e nice_stats[node_type] = {} if total != 0: try: - documented = total - stats["undocumented_" + node_type] + undocumented_node: int = stats["undocumented_" + node_type] # type: ignore + documented = total - undocumented_node percent = (documented * 100.0) / total nice_stats[node_type]["percent_documented"] = f"{percent:.2f}" except KeyError: nice_stats[node_type]["percent_documented"] = "NC" try: - percent = (stats["badname_" + node_type] * 100.0) / total + badname_node: int = stats["badname_" + node_type] # type: ignore + percent = (badname_node * 100.0) / total nice_stats[node_type]["percent_badname"] = f"{percent:.2f}" except KeyError: nice_stats[node_type]["percent_badname"] = "NC" lines = ["type", "number", "old number", "difference", "%documented", "%badname"] for node_type in ("module", "class", "method", "function"): new = stats[node_type] - old = old_stats.get(node_type, None) + old: Optional[Union[str, int]] = old_stats.get(node_type, None) # type: ignore if old is not None: diff_str = lint_utils.diff_string(old, new) else: @@ -1082,7 +1089,7 @@ class BasicChecker(_BasicChecker): def __init__(self, linter): _BasicChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} self._tryfinallys = None def open(self): @@ -1159,13 +1166,13 @@ def _check_using_constant_test(self, node, test): def visit_module(self, _: nodes.Module) -> None: """check module name, docstring and required arguments""" - self.stats["module"] += 1 + self.stats["module"] += 1 # type: ignore def visit_classdef(self, _: nodes.ClassDef) -> None: """check module name, docstring and redefinition increment branch counter """ - self.stats["class"] += 1 + self.stats["class"] += 1 # type: ignore @utils.check_messages( "pointless-statement", "pointless-string-statement", "expression-not-assigned" @@ -1304,7 +1311,7 @@ def visit_functiondef(self, node: nodes.FunctionDef) -> None: """check function name, docstring, arguments, redefinition, variable names, max locals """ - self.stats["method" if node.is_method() else "function"] += 1 + self.stats["method" if node.is_method() else "function"] += 1 # type: ignore self._check_dangerous_default(node) visit_asyncfunctiondef = visit_functiondef @@ -2040,7 +2047,7 @@ def _raise_name_warning( ) self.add_message(warning, node=node, args=args, confidence=confidence) - self.stats["badname_" + node_type] += 1 + self.stats["badname_" + node_type] += 1 # type: ignore def _name_allowed_by_regex(self, name: str) -> bool: return name in self.config.good_names or any( diff --git a/pylint/checkers/base_checker.py b/pylint/checkers/base_checker.py index 5dfd7ea782..791a3cb1e0 100644 --- a/pylint/checkers/base_checker.py +++ b/pylint/checkers/base_checker.py @@ -24,6 +24,7 @@ from pylint.exceptions import InvalidMessageError from pylint.interfaces import UNDEFINED, IRawChecker, ITokenChecker, implements from pylint.message.message_definition import MessageDefinition +from pylint.typing import CheckerStats from pylint.utils import get_rst_section, get_rst_title @@ -51,6 +52,7 @@ def __init__(self, linter=None): self.name = self.name.lower() OptionsProviderMixIn.__init__(self) self.linter = linter + self.stats: CheckerStats = {} def __gt__(self, other): """Permit to sort a list of Checker by name.""" diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index 9777b568e6..362d299de1 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -35,6 +35,7 @@ from pylint.checkers import BaseChecker from pylint.checkers.utils import check_messages from pylint.interfaces import IAstroidChecker +from pylint.typing import CheckerStats MSGS = { # pylint: disable=consider-using-namedtuple-or-dataclass "R0901": ( @@ -391,7 +392,7 @@ class MisdesignChecker(BaseChecker): def __init__(self, linter=None): BaseChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} self._returns = None self._branches = None self._stmts = None diff --git a/pylint/checkers/imports.py b/pylint/checkers/imports.py index 7e0cda26dc..8b413253fc 100644 --- a/pylint/checkers/imports.py +++ b/pylint/checkers/imports.py @@ -68,6 +68,7 @@ from pylint.interfaces import IAstroidChecker from pylint.lint import PyLinter from pylint.reporters.ureports.nodes import Paragraph, VerbatimText, VNode +from pylint.typing import CheckerStats from pylint.utils import IsortDriver, get_global_option @@ -423,7 +424,7 @@ def __init__( self, linter: PyLinter = None ): # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941 BaseChecker.__init__(self, linter) - self.stats: Dict[Any, Any] = {} + self.stats: CheckerStats = {} self.import_graph: collections.defaultdict = collections.defaultdict(set) self._imports_stack: List[Tuple[Any, Any]] = [] self._first_non_import_node = None @@ -839,9 +840,8 @@ def _add_imported_module( self._module_pkg[context_name] = context_name.rsplit(".", 1)[0] # handle dependencies - importedmodnames = self.stats["dependencies"].setdefault( - importedmodname, set() - ) + dependencies_stat: Dict[str, Union[Set]] = self.stats["dependencies"] # type: ignore + importedmodnames = dependencies_stat.setdefault(importedmodname, set()) if context_name not in importedmodnames: importedmodnames.add(context_name) diff --git a/pylint/checkers/raw_metrics.py b/pylint/checkers/raw_metrics.py index 028c68e7ab..cc3f2729d6 100644 --- a/pylint/checkers/raw_metrics.py +++ b/pylint/checkers/raw_metrics.py @@ -15,27 +15,32 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import tokenize -from typing import Any +from typing import Any, Optional, Union from pylint.checkers import BaseTokenChecker from pylint.exceptions import EmptyReportError from pylint.interfaces import ITokenChecker from pylint.reporters.ureports.nodes import Table +from pylint.typing import CheckerStats from pylint.utils import diff_string -def report_raw_stats(sect, stats, old_stats): +def report_raw_stats( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """calculate percentage of code / doc / comment / empty""" - total_lines = stats["total_lines"] + total_lines: int = stats["total_lines"] # type: ignore if not total_lines: raise EmptyReportError() sect.description = f"{total_lines} lines have been analyzed" lines = ["type", "number", "%", "previous", "difference"] for node_type in ("code", "docstring", "comment", "empty"): key = node_type + "_lines" - total = stats[key] + total: int = stats[key] # type: ignore percent = float(total * 100) / total_lines - old = old_stats.get(key, None) + old: Optional[Union[int, str]] = old_stats.get(key, None) # type: ignore if old is not None: diff_str = diff_string(old, total) else: @@ -66,7 +71,7 @@ class RawMetricsChecker(BaseTokenChecker): def __init__(self, linter): BaseTokenChecker.__init__(self, linter) - self.stats = None + self.stats: CheckerStats = {} def open(self): """init statistics""" diff --git a/pylint/checkers/similar.py b/pylint/checkers/similar.py index 4ceb5d6603..2a28e7d6c5 100644 --- a/pylint/checkers/similar.py +++ b/pylint/checkers/similar.py @@ -73,6 +73,7 @@ from pylint.checkers import BaseChecker, MapReduceMixin, table_lines_from_stats from pylint.interfaces import IRawChecker from pylint.reporters.ureports.nodes import Table +from pylint.typing import CheckerStats from pylint.utils import decoding_stream DEFAULT_MIN_SIMILARITY_LINE = 4 @@ -721,7 +722,11 @@ def real_lines(self): } -def report_similarities(sect, stats, old_stats): +def report_similarities( + sect, + stats: CheckerStats, + old_stats: CheckerStats, +): """make a layout with some stats about duplication""" lines = ["", "now", "previous", "difference"] lines += table_lines_from_stats( @@ -804,7 +809,7 @@ def __init__(self, linter=None) -> None: ignore_imports=self.config.ignore_imports, ignore_signatures=self.config.ignore_signatures, ) - self.stats = None + self.stats: CheckerStats = {} def set_option(self, optname, value, action=None, optdict=None): """method called to set an option (registered in the options list) diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index ad6830721d..2f3d0dd050 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -3,10 +3,15 @@ import collections import functools +from typing import TYPE_CHECKING, Dict, List, Union from pylint import reporters from pylint.lint.utils import _patch_sys_path from pylint.message import Message +from pylint.typing import CheckerStats + +if TYPE_CHECKING: + from typing import Counter # typing.Counter added in Python 3.6.1 try: import multiprocessing @@ -30,20 +35,20 @@ def _get_new_args(message): return (message.msg_id, message.symbol, location, message.msg, message.confidence) -def _merge_stats(stats): - merged = {} - by_msg = collections.Counter() +def _merge_stats(stats: List[CheckerStats]): + merged: CheckerStats = {} + by_msg: "Counter[str]" = collections.Counter() for stat in stats: - message_stats = stat.pop("by_msg", {}) + message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore by_msg.update(message_stats) for key, item in stat.items(): if key not in merged: merged[key] = item elif isinstance(item, dict): - merged[key].update(item) + merged[key].update(item) # type: ignore else: - merged[key] = merged[key] + item + merged[key] = merged[key] + item # type: ignore merged["by_msg"] = by_msg return merged diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 54614ad890..d731e71b3f 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -31,6 +31,7 @@ ) from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn from pylint.reporters.ureports import nodes as report_nodes +from pylint.typing import CheckerStats from pylint.utils import ASTWalker, FileState, utils from pylint.utils.pragma_parser import ( OPTION_PO, @@ -502,7 +503,7 @@ def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None): self.file_state = FileState() self.current_name = None self.current_file = None - self.stats = None + self.stats: CheckerStats = {} self.fail_on_symbols = [] # init options self._external_opts = options @@ -729,8 +730,10 @@ def enable_fail_on_messages(self): self.fail_on_symbols.append(msg.symbol) def any_fail_on_issues(self): - return self.stats is not None and any( - x in self.fail_on_symbols for x in self.stats["by_msg"] + return ( + self.stats + and self.stats.get("by_msg") is not None + and any(x in self.fail_on_symbols for x in self.stats["by_msg"]) ) def disable_noerror_messages(self): diff --git a/pylint/lint/report_functions.py b/pylint/lint/report_functions.py index 21cb3b8245..12c4b03352 100644 --- a/pylint/lint/report_functions.py +++ b/pylint/lint/report_functions.py @@ -2,12 +2,18 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections +from typing import DefaultDict, Dict, List, Tuple, Union from pylint import checkers, exceptions from pylint.reporters.ureports import nodes as report_nodes +from pylint.typing import CheckerStats -def report_total_messages_stats(sect, stats, previous_stats): +def report_total_messages_stats( + sect, + stats: CheckerStats, + previous_stats: CheckerStats, +): """make total errors / warnings report""" lines = ["type", "number", "previous", "difference"] lines += checkers.table_lines_from_stats( @@ -16,14 +22,19 @@ def report_total_messages_stats(sect, stats, previous_stats): sect.append(report_nodes.Table(children=lines, cols=4, rheaders=1)) -def report_messages_stats(sect, stats, _): +def report_messages_stats( + sect, + stats: CheckerStats, + _: CheckerStats, +): """make messages type report""" if not stats["by_msg"]: # don't print this report when we didn't detected any errors raise exceptions.EmptyReportError() - in_order = sorted( + by_msg_stats: Dict[str, int] = stats["by_msg"] # type: ignore + in_order: List[Tuple[int, str]] = sorted( (value, msg_id) - for msg_id, value in stats["by_msg"].items() + for msg_id, value in by_msg_stats.items() if not msg_id.startswith("I") ) in_order.reverse() @@ -33,16 +44,23 @@ def report_messages_stats(sect, stats, _): sect.append(report_nodes.Table(children=lines, cols=2, rheaders=1)) -def report_messages_by_module_stats(sect, stats, _): +def report_messages_by_module_stats( + sect, + stats: CheckerStats, + _: CheckerStats, +): """make errors / warnings by modules report""" - if len(stats["by_module"]) == 1: + module_stats: Dict[str, Dict[str, int]] = stats["by_module"] # type: ignore + if len(module_stats) == 1: # don't print this report when we are analysing a single module raise exceptions.EmptyReportError() - by_mod = collections.defaultdict(dict) + by_mod: DefaultDict[str, Dict[str, Union[int, float]]] = collections.defaultdict( + dict + ) for m_type in ("fatal", "error", "warning", "refactor", "convention"): - total = stats[m_type] - for module in stats["by_module"].keys(): - mod_total = stats["by_module"][module][m_type] + total: int = stats[m_type] # type: ignore + for module in module_stats.keys(): + mod_total = module_stats[module][m_type] percent = 0 if total == 0 else float((mod_total) * 100) / total by_mod[module][m_type] = percent sorted_result = [] diff --git a/pylint/reporters/base_reporter.py b/pylint/reporters/base_reporter.py index fda99f4ef9..4cf5d5895d 100644 --- a/pylint/reporters/base_reporter.py +++ b/pylint/reporters/base_reporter.py @@ -6,6 +6,7 @@ from typing import List from pylint.message import Message +from pylint.typing import CheckerStats class BaseReporter: @@ -41,7 +42,7 @@ def writeln(self, string=""): def display_reports(self, layout): """display results encapsulated in the layout tree""" self.section = 0 - if hasattr(layout, "report_id"): + if layout.report_id: layout.children[0].children[0].data += f" ({layout.report_id})" self._display(layout) @@ -65,5 +66,9 @@ def display_messages(self, layout): def on_set_current_module(self, module, filepath): """Hook called when a module starts to be analysed.""" - def on_close(self, stats, previous_stats): + def on_close( + self, + stats: CheckerStats, + previous_stats: CheckerStats, + ) -> None: """Hook called when a module finished analyzing.""" diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py index a4dbae53b7..245c10f79c 100644 --- a/pylint/reporters/multi_reporter.py +++ b/pylint/reporters/multi_reporter.py @@ -3,12 +3,13 @@ import os -from typing import IO, Any, AnyStr, Callable, List, Mapping, Optional, Union +from typing import IO, Any, AnyStr, Callable, List, Optional, Union from pylint.interfaces import IReporter from pylint.message import Message from pylint.reporters.base_reporter import BaseReporter from pylint.reporters.ureports.nodes import BaseLayout +from pylint.typing import CheckerStats AnyFile = IO[AnyStr] AnyPath = Union[str, bytes, os.PathLike] @@ -95,8 +96,8 @@ def on_set_current_module(self, module: str, filepath: Optional[AnyPath]) -> Non def on_close( self, - stats: Mapping[Any, Any], - previous_stats: Mapping[Any, Any], + stats: CheckerStats, + previous_stats: CheckerStats, ) -> None: """hook called when a module finished analyzing""" for rep in self._sub_reporters: diff --git a/pylint/reporters/reports_handler_mix_in.py b/pylint/reporters/reports_handler_mix_in.py index 914556ef48..450d383f51 100644 --- a/pylint/reporters/reports_handler_mix_in.py +++ b/pylint/reporters/reports_handler_mix_in.py @@ -2,9 +2,14 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections +from typing import TYPE_CHECKING, Any from pylint.exceptions import EmptyReportError from pylint.reporters.ureports.nodes import Section +from pylint.typing import CheckerStats + +if TYPE_CHECKING: + from pylint.lint.pylinter import PyLinter class ReportsHandlerMixIn: @@ -49,7 +54,11 @@ def report_is_enabled(self, reportid): """ return self._reports_state.get(reportid, True) - def make_reports(self, stats, old_stats): + def make_reports( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter + self: "PyLinter", + stats: CheckerStats, + old_stats: CheckerStats, + ): """render registered reports""" sect = Section("Report", f"{self.stats['statement']} statements analysed.") for checker in self.report_order(): @@ -65,7 +74,9 @@ def make_reports(self, stats, old_stats): sect.append(report_sect) return sect - def add_stats(self, **kwargs): + def add_stats( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter + self: "PyLinter", **kwargs: Any + ) -> CheckerStats: """add some stats entries to the statistic dictionary raise an AssertionError if there is a key conflict """ diff --git a/pylint/reporters/ureports/nodes.py b/pylint/reporters/ureports/nodes.py index 3f7842a18e..c5c14737da 100644 --- a/pylint/reporters/ureports/nodes.py +++ b/pylint/reporters/ureports/nodes.py @@ -14,6 +14,7 @@ A micro report is a tree of layout and content objects. """ +from typing import Optional class VNode: @@ -126,6 +127,7 @@ def __init__(self, title=None, description=None, **kwargs): self.insert(0, Paragraph([Text(description)])) if title: self.insert(0, Title(children=(title,))) + self.report_id: Optional[str] = None class EvaluationSection(Section): diff --git a/pylint/testutils/unittest_linter.py b/pylint/testutils/unittest_linter.py index 2d0a4f2c3f..b75592ce6a 100644 --- a/pylint/testutils/unittest_linter.py +++ b/pylint/testutils/unittest_linter.py @@ -3,6 +3,7 @@ from pylint.testutils.global_test_linter import linter from pylint.testutils.output_line import Message +from pylint.typing import CheckerStats class UnittestLinter: @@ -12,7 +13,7 @@ class UnittestLinter: def __init__(self): self._messages = [] - self.stats = {} + self.stats: CheckerStats = {} def release_messages(self): try: diff --git a/pylint/typing.py b/pylint/typing.py new file mode 100644 index 0000000000..9ce1c587f7 --- /dev/null +++ b/pylint/typing.py @@ -0,0 +1,14 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +"""A collection of typing utilities.""" +from typing import TYPE_CHECKING, Dict, List, Union + +if TYPE_CHECKING: + from typing import Counter # typing.Counter added in Python 3.6.1 + + +# The base type of the "stats" attribute of a checker +CheckerStats = Dict[ + str, Union[int, "Counter[str]", List, Dict[str, Union[int, str, Dict[str, int]]]] +] diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py index 18656ad740..c31cbdba06 100644 --- a/tests/lint/unittest_lint.py +++ b/tests/lint/unittest_lint.py @@ -48,7 +48,7 @@ from os import chdir, getcwd from os.path import abspath, basename, dirname, isdir, join, sep from shutil import rmtree -from typing import Iterable, Iterator, List, Optional, Tuple +from typing import Dict, Iterable, Iterator, List, Optional, Tuple import platformdirs import pytest @@ -828,7 +828,8 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None: linter = init_linter linter.check(os.path.join(os.path.dirname(__file__), "data")) - for module, module_stats in linter.stats["by_module"].items(): + by_module_stats: Dict[str, Dict[str, int]] = linter.stats["by_module"] # type: ignore + for module, module_stats in by_module_stats.items(): linter2 = init_linter if module == "data": diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py index 102c0f1225..66236ac8dc 100644 --- a/tests/test_check_parallel.py +++ b/tests/test_check_parallel.py @@ -24,6 +24,7 @@ from pylint.lint.parallel import _worker_initialize as worker_initialize from pylint.lint.parallel import check_parallel from pylint.testutils import GenericTestReporter as Reporter +from pylint.typing import CheckerStats def _gen_file_data(idx: int = 0) -> Tuple[str, str, str]: @@ -101,7 +102,7 @@ def __init__(self, linter: PyLinter) -> None: super().__init__(linter) self.data: List[str] = [] self.linter = linter - self.stats = None + self.stats: CheckerStats = {} def open(self) -> None: """init the checkers: reset statistics information""" diff --git a/tests/test_regr.py b/tests/test_regr.py index 793a196129..12dd03406a 100644 --- a/tests/test_regr.py +++ b/tests/test_regr.py @@ -23,7 +23,7 @@ import os import sys from os.path import abspath, dirname, join -from typing import Iterator +from typing import Dict, Iterator import astroid import pytest @@ -111,12 +111,14 @@ def modify_path() -> Iterator: def test_check_package___init__(finalize_linter: PyLinter) -> None: filename = "package.__init__" finalize_linter.check(filename) - checked = list(finalize_linter.stats["by_module"].keys()) + by_module_stats: Dict[str, Dict[str, int]] = finalize_linter.stats["by_module"] # type: ignore + checked = list(by_module_stats.keys()) assert checked == [filename] os.chdir(join(REGR_DATA, "package")) finalize_linter.check("__init__") - checked = list(finalize_linter.stats["by_module"].keys()) + by_module_stats: Dict[str, Dict[str, int]] = finalize_linter.stats["by_module"] # type: ignore + checked = list(by_module_stats.keys()) assert checked == ["__init__"]