From 598c9bfc6bdad608ed3c4074ada823c83244714d Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 27 Mar 2019 15:29:15 -0700 Subject: [PATCH 1/3] Add a get_additional_deps hook to Plugin to support django-stubs django-stubs needs to be able to add in additional dependencies to modules and is currently using monkeypatching to do it. That is 1) not great and 2) breaks under mypyc. Add a hook to support generating additional dependencies from a plugin. --- mypy/build.py | 4 +++- mypy/interpreted_plugin.py | 5 ++++- mypy/plugin.py | 24 +++++++++++++++++++++++- test-data/unit/check-custom-plugin.test | 15 ++++++++++++++- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c6c44fd6ead8..8391c34e48ec 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1967,7 +1967,9 @@ def compute_dependencies(self) -> None: dependencies = [] priorities = {} # type: Dict[str, int] # id -> priority dep_line_map = {} # type: Dict[str, int] # id -> line - for pri, id, line in manager.all_imported_modules_in_file(self.tree): + dep_entries = (manager.all_imported_modules_in_file(self.tree) + + self.manager.plugin.get_additional_deps(self.tree)) + for pri, id, line in dep_entries: priorities[id] = min(pri, priorities.get(id, PRI_ALL)) if id == self.id: continue diff --git a/mypy/interpreted_plugin.py b/mypy/interpreted_plugin.py index e95fbe1e3260..981e8eb350d6 100644 --- a/mypy/interpreted_plugin.py +++ b/mypy/interpreted_plugin.py @@ -1,6 +1,6 @@ """Hack for handling non-mypyc compiled plugins with a mypyc-compiled mypy""" -from typing import Optional, Callable, Any, Dict +from typing import Optional, Callable, Any, Dict, List, Tuple from mypy.options import Options from mypy.types import Type, CallableType from mypy.nodes import SymbolTableNode, MypyFile @@ -39,6 +39,9 @@ def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]: assert self._modules is not None return lookup_fully_qualified(fullname, self._modules) + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + return [] + def get_type_analyze_hook(self, fullname: str ) -> Optional[Callable[['mypy.plugin.AnalyzeTypeContext'], Type]]: return None diff --git a/mypy/plugin.py b/mypy/plugin.py index ca66c47999dd..8ca685fc9e83 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -371,6 +371,19 @@ def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]: assert self._modules is not None return lookup_fully_qualified(fullname, self._modules) + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + """Customize dependencies for a module. + + This hook allows adding in new dependencies for a module. It is called + after parsing a file but before analysis. + + Returns a list of (priority, module name, line number) tuples. + + Priorities are defined in mypy.build (but maybe shouldn't be). + 10 is a good choice for priority. + """ + return [] + def get_type_analyze_hook(self, fullname: str ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: """Customize behaviour of the type analyzer for given full names. @@ -395,7 +408,7 @@ def get_function_hook(self, fullname: str """Adjust the return type of a function call. This method is called after type checking a call. Plugin may adjust the return - type inferred by mypy, and/or emmit some error messages. Note, this hook is also + type inferred by mypy, and/or emit some error messages. Note, this hook is also called for class instantiation calls, so that in this example: from lib import Class, do_stuff @@ -561,6 +574,9 @@ def set_modules(self, modules: Dict[str, MypyFile]) -> None: def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]: return self.plugin.lookup_fully_qualified(fullname) + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + return self.plugin.get_additional_deps(file) + def get_type_analyze_hook(self, fullname: str ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: return self.plugin.get_type_analyze_hook(fullname) @@ -626,6 +642,12 @@ def set_modules(self, modules: Dict[str, MypyFile]) -> None: for plugin in self._plugins: plugin.set_modules(modules) + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + deps = [] + for plugin in self._plugins: + deps.extend(plugin.get_additional_deps(file)) + return deps + def get_type_analyze_hook(self, fullname: str ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index b810b8ba46b2..905f2a029eff 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -71,7 +71,6 @@ tmp/mypy.ini:2: error: Can't find plugin 'tmp/missing.py' plugins=missing [out] tmp/mypy.ini:2: error: Error importing plugin 'missing' ---' (work around syntax highlighting) [case testMultipleSectionsDefinePlugin] # flags: --config-file tmp/mypy.ini @@ -585,3 +584,17 @@ reveal_type(instance(2)) # E: Revealed type is 'builtins.float*' [file mypy.ini] [[mypy] plugins=/test-data/unit/plugins/callable_instance.py + + +[case testPluginDependencies] +# flags: --config-file tmp/mypy.ini + +# The top level file here doesn't do anything, but the plugin should add +# a dependency on err that will cause err to be processed and an error reported. + +[file err.py] +1 + 'lol' # E: Unsupported operand types for + ("int" and "str") + +[file mypy.ini] +[[mypy] +plugins=/test-data/unit/plugins/depshook.py From b8f0036c4262425a74a8a18d874039d1522c0188 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 27 Mar 2019 16:06:18 -0700 Subject: [PATCH 2/3] add in a test file --- test-data/unit/plugins/depshook.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test-data/unit/plugins/depshook.py diff --git a/test-data/unit/plugins/depshook.py b/test-data/unit/plugins/depshook.py new file mode 100644 index 000000000000..99a6fd14dc02 --- /dev/null +++ b/test-data/unit/plugins/depshook.py @@ -0,0 +1,15 @@ +from typing import Optional, Callable, List, Tuple + +from mypy.plugin import Plugin +from mypy.nodes import MypyFile + + +class DepsPlugin(Plugin): + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + if file.fullname() == '__main__': + return [(10, 'err', -1)] + return [] + + +def plugin(version): + return DepsPlugin From 66737f1c833f82de3d4e9ae8e3c45514d47b452c Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Thu, 28 Mar 2019 16:33:41 -0700 Subject: [PATCH 3/3] some tweaks --- mypy/plugin.py | 8 ++++++-- test-data/unit/check-custom-plugin.test | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 8ca685fc9e83..1e46b3c3f52f 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -374,11 +374,15 @@ def lookup_fully_qualified(self, fullname: str) -> Optional[SymbolTableNode]: def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: """Customize dependencies for a module. - This hook allows adding in new dependencies for a module. It is called - after parsing a file but before analysis. + This hook allows adding in new dependencies for a module. It + is called after parsing a file but before analysis. This can + be useful if a library has dependencies that are dynamic based + on configuration information, for example. Returns a list of (priority, module name, line number) tuples. + The line number can be -1 when there is not a known real line number. + Priorities are defined in mypy.build (but maybe shouldn't be). 10 is a good choice for priority. """ diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index 905f2a029eff..44ac81cbc865 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -585,7 +585,6 @@ reveal_type(instance(2)) # E: Revealed type is 'builtins.float*' [[mypy] plugins=/test-data/unit/plugins/callable_instance.py - [case testPluginDependencies] # flags: --config-file tmp/mypy.ini