diff --git a/CHANGELOG.md b/CHANGELOG.md index 663cc2a..b1ba02e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Cache namespace results and minimize kwarg checks by grouping similar rules - ~500% speed up ([#18](https://github.com/dlint-py/dlint/issues/18)) + ### Fixed - The `--print-dlint-linters` flag on Windows ([#17](https://github.com/dlint-py/dlint/issues/17)) diff --git a/dlint/linters/helpers/bad_kwarg_use.py b/dlint/linters/helpers/bad_kwarg_use.py index 687ea38..e45b5c7 100644 --- a/dlint/linters/helpers/bad_kwarg_use.py +++ b/dlint/linters/helpers/bad_kwarg_use.py @@ -9,6 +9,7 @@ import abc import ast +import itertools from .. import base from ... import tree @@ -20,6 +21,56 @@ class BadKwargUseLinter(base.BaseLinter, util.ABC): lint rules that block bad kwarg use. """ + def __init__(self, *args, **kwargs): + self.minimized_bad_kwarg_func = None + + module_path_grouped = [ + (k, list(v)) + for k, v in itertools.groupby( + sorted(self.kwargs, key=lambda k: k["module_path"]), + key=lambda k: k["module_path"] + ) + ] + + def minimized_illegal_module_imported(module_path, node): + return any( + self.namespace.illegal_module_imported( + module_path, + kwarg["module_path"] + ) + and kwarg["predicate"](node, kwarg["kwarg_name"]) + for illegal_module_path, kwargs in module_path_grouped + for kwarg in kwargs + ) + + kwarg_predicate_grouped = [ + (k, list(v)) + for k, v in itertools.groupby( + sorted(self.kwargs, key=lambda k: (k["kwarg_name"], id(k["predicate"]))), + key=lambda k: (k["kwarg_name"], id(k["predicate"])) + ) + ] + + def minimized_kwarg_predicate(module_path, node): + return any( + self.namespace.illegal_module_imported( + module_path, + kwarg["module_path"] + ) + and kwarg["predicate"](node, kwarg["kwarg_name"]) + for kwarg_predicate_tuple, kwargs in kwarg_predicate_grouped + for kwarg in kwargs + ) + + # Minimize kwarg checks by grouping similar rules + if (len(kwarg_predicate_grouped) < len(self.kwargs) + and len(module_path_grouped) == len(self.kwargs)): + self.minimized_bad_kwarg_func = minimized_kwarg_predicate + else: + self.minimized_bad_kwarg_func = minimized_illegal_module_imported + + super(BadKwargUseLinter, self).__init__(*args, **kwargs) + @property @abc.abstractmethod def kwargs(self): @@ -45,15 +96,9 @@ def visit_Call(self, node): if not isinstance(node.func, (ast.Attribute, ast.Name)): return - bad_kwarg = any( - ( - self.namespace.illegal_module_imported( - tree.module_path_str(node.func), - kwarg["module_path"] - ) - and kwarg["predicate"](node, kwarg["kwarg_name"]) - ) - for kwarg in self.kwargs + bad_kwarg = self.minimized_bad_kwarg_func( + tree.module_path_str(node.func), + node ) if bad_kwarg: diff --git a/dlint/namespace.py b/dlint/namespace.py index af55ef5..5b77608 100644 --- a/dlint/namespace.py +++ b/dlint/namespace.py @@ -10,6 +10,17 @@ import ast import copy +try: + from functools import lru_cache +except ImportError: + # Sorry Python 2 users, it's time to upgrade + def lru_cache(*args, **kwargs): + def decorator(function): + def noop(*inner_args, **inner_kwargs): + return function(*inner_args, **inner_kwargs) + return noop + return decorator + from . import util @@ -37,6 +48,7 @@ def from_module_node(cls, module_node): return cls(imports, from_imports) + @lru_cache(maxsize=1024) def name_imported(self, name): def alias_includes_name(alias): return ( @@ -58,6 +70,7 @@ def asname_to_name(self, asname): return None + @lru_cache(maxsize=1024) def illegal_module_imported(self, module_path, illegal_module_path): modules = module_path.split('.') illegal_modules = illegal_module_path.split('.') diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 73bd11a..2166fa1 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -20,6 +20,11 @@ def pytest_addoption(parser): type=argparse.FileType("r"), help="Benchmark Dlint against this Python file." ) + parser.addoption( + "--benchmark-group-base-class", + action="store_true", + help="Group Dlint benchmark results by base class." + ) @pytest.fixture @@ -31,3 +36,8 @@ def benchmark_py_file(request): fd.seek(0) return ast.parse(fd.read()) + + +@pytest.fixture +def benchmark_group_base_class(request): + return request.config.getoption("--benchmark-group-base-class") diff --git a/tests/test_benchmark/test_benchmark.py b/tests/test_benchmark/test_benchmark.py index b3980ed..57334c2 100644 --- a/tests/test_benchmark/test_benchmark.py +++ b/tests/test_benchmark/test_benchmark.py @@ -47,7 +47,10 @@ def test_benchmark_run(benchmark_py_file, benchmark): sorted(extension.dlint.linters.ALL, key=lambda l: l._code), ids=lambda l: "{}-{}".format(l._code, l.__name__) ) -def test_benchmark_individual(benchmark_py_file, benchmark, linter): +def test_benchmark_individual(benchmark_py_file, benchmark_group_base_class, benchmark, linter): + if benchmark_group_base_class: + benchmark.group = str(linter.__bases__) + ext_class = get_single_linter_extension(linter) ext = ext_class(benchmark_py_file, "unused")