Don't suggest incompatible stub packages (#10610)

Keep track of supported Python versions of legacy bundled packages, and
only suggest a stub package if the major Python version is compatible.

Fixes #10602.

mypy/build.py

@@ -58,7 +58,7 @@
 from mypy.renaming import VariableRenameVisitor
 from mypy.config_parser import parse_mypy_comments
 from mypy.freetree import free_tree
-from mypy.stubinfo import legacy_bundled_packages
+from mypy.stubinfo import legacy_bundled_packages, is_legacy_bundled_package
 from mypy import errorcodes as codes
 
 
@@ -2449,7 +2449,9 @@ def find_module_and_diagnose(manager: BuildManager,
         # otherwise updating mypy can silently result in new false
         # negatives.
         global_ignore_missing_imports = manager.options.ignore_missing_imports
-        if ((top_level in legacy_bundled_packages or second_level in legacy_bundled_packages)
+        py_ver = options.python_version[0]
+        if ((is_legacy_bundled_package(top_level, py_ver)
+                or is_legacy_bundled_package(second_level, py_ver))
                 and global_ignore_missing_imports
                 and not options.ignore_missing_imports_per_module):
             ignore_missing_imports = False
@@ -2558,10 +2560,10 @@ def module_not_found(manager: BuildManager, line: int, caller_state: State,
             top_level = second_level
         for note in notes:
             if '{stub_dist}' in note:
-                note = note.format(stub_dist=legacy_bundled_packages[top_level])
+                note = note.format(stub_dist=legacy_bundled_packages[top_level].name)
             errors.report(line, 0, note, severity='note', only_once=True, code=codes.IMPORT)
         if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
-            manager.missing_stub_packages.add(legacy_bundled_packages[top_level])
+            manager.missing_stub_packages.add(legacy_bundled_packages[top_level].name)
     errors.set_import_context(save_import_context)
 
 

mypy/modulefinder.py

@@ -17,7 +17,7 @@
 
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
-from mypy.stubinfo import legacy_bundled_packages
+from mypy.stubinfo import is_legacy_bundled_package
 from mypy import sitepkgs
 
 # Paths to be searched in find_module().
@@ -136,7 +136,7 @@ def __init__(self,
         if options:
             custom_typeshed_dir = options.custom_typeshed_dir
         self.stdlib_py_versions = load_stdlib_py_versions(custom_typeshed_dir)
-        self.python2 = options and options.python_version[0] == 2
+        self.python_major_ver = 3 if options is None else options.python_version[0]
 
     def clear(self) -> None:
         self.results.clear()
@@ -187,7 +187,7 @@ def get_toplevel_possibilities(self, lib_path: Tuple[str, ...], id: str) -> List
                 name = os.path.splitext(name)[0]
                 components.setdefault(name, []).append(dir)
 
-        if self.python2:
+        if self.python_major_ver == 2:
             components = {id: filter_redundant_py2_dirs(dirs)
                           for id, dirs in components.items()}
 
@@ -230,8 +230,8 @@ def _find_module_non_stub_helper(self, components: List[str],
             elif not plausible_match and (self.fscache.isdir(dir_path)
                                           or self.fscache.isfile(dir_path + ".py")):
                 plausible_match = True
-        if (components[0] in legacy_bundled_packages
-                or '.'.join(components[:2]) in legacy_bundled_packages):
+        if (is_legacy_bundled_package(components[0], self.python_major_ver)
+                or is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver)):
             return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
         elif plausible_match:
             return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
@@ -280,7 +280,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
         for pkg_dir in self.search_paths.package_path:
             stub_name = components[0] + '-stubs'
             stub_dir = os.path.join(pkg_dir, stub_name)
-            if self.python2:
+            if self.python_major_ver == 2:
                 alt_stub_name = components[0] + '-python2-stubs'
                 alt_stub_dir = os.path.join(pkg_dir, alt_stub_name)
                 if fscache.isdir(alt_stub_dir):
@@ -348,7 +348,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
             for extension in PYTHON_EXTENSIONS:
                 path = base_path + sepinit + extension
                 suffix = '-stubs'
-                if self.python2:
+                if self.python_major_ver == 2:
                     if os.path.isdir(base_path + '-python2-stubs'):
                         suffix = '-python2-stubs'
                 path_stubs = base_path + suffix + sepinit + extension
@@ -432,7 +432,7 @@ def _is_compatible_stub_package(self, stub_dir: str) -> bool:
             import toml
             with open(metadata_fnam, 'r') as f:
                 metadata = toml.load(f)
-            if self.python2:
+            if self.python_major_ver == 2:
                 return bool(metadata.get('python2', False))
             else:
                 return bool(metadata.get('python3', True))

mypy/stubinfo.py

@@ -1,83 +1,100 @@
+from typing import Optional
+
+
+class StubInfo:
+    def __init__(self, name: str, py_version: Optional[int] = None) -> None:
+        self.name = name
+        # If None, compatible with py2+py3, if 2/3, only compatible with py2/py3
+        self.py_version = py_version
+
+
+def is_legacy_bundled_package(prefix: str, py_version: int) -> bool:
+    if prefix not in legacy_bundled_packages:
+        return False
+    package_ver = legacy_bundled_packages[prefix].py_version
+    return package_ver is None or package_ver == py_version
+
+
 # Stubs for these third-party packages used to be shipped with mypy.
 #
 # Map package name to PyPI stub distribution name.
 #
 # Package name can have one or two components ('a' or 'a.b').
 legacy_bundled_packages = {
-    'aiofiles': 'types-aiofiles',
-    'atomicwrites': 'types-atomicwrites',
-    'attr': 'types-attrs',
-    'backports': 'types-backports',
-    'backports_abc': 'types-backports_abc',
-    'bleach': 'types-bleach',
-    'boto': 'types-boto',
-    'cachetools': 'types-cachetools',
-    'certifi': 'types-certifi',
-    'characteristic': 'types-characteristic',
-    'chardet': 'types-chardet',
-    'click': 'types-click',
-    'click_spinner': 'types-click-spinner',
-    'concurrent': 'types-futures',
-    'contextvars': 'types-contextvars',
-    'croniter': 'types-croniter',
-    'cryptography': 'types-cryptography',
-    'dataclasses': 'types-dataclasses',
-    'dateparser': 'types-dateparser',
-    'datetimerange': 'types-DateTimeRange',
-    'dateutil': 'types-python-dateutil',
-    'decorator': 'types-decorator',
-    'deprecated': 'types-Deprecated',
-    'docutils': 'types-docutils',
-    'emoji': 'types-emoji',
-    'enum': 'types-enum34',
-    'fb303': 'types-fb303',
-    'filelock': 'types-filelock',
-    'first': 'types-first',
-    'flask': 'types-Flask',
-    'freezegun': 'types-freezegun',
-    'frozendict': 'types-frozendict',
-    'geoip2': 'types-geoip2',
-    'gflags': 'types-python-gflags',
-    'google.protobuf': 'types-protobuf',
-    'ipaddress': 'types-ipaddress',
-    'itsdangerous': 'types-itsdangerous',
-    'jinja2': 'types-Jinja2',
-    'jwt': 'types-jwt',
-    'kazoo': 'types-kazoo',
-    'markdown': 'types-Markdown',
-    'markupsafe': 'types-MarkupSafe',
-    'maxminddb': 'types-maxminddb',
-    'mock': 'types-mock',
-    'OpenSSL': 'types-openssl-python',
-    'orjson': 'types-orjson',
-    'paramiko': 'types-paramiko',
-    'pathlib2': 'types-pathlib2',
-    'pkg_resources': 'types-pkg_resources',
-    'polib': 'types-polib',
-    'pycurl': 'types-pycurl',
-    'pymssql': 'types-pymssql',
-    'pymysql': 'types-PyMySQL',
-    'pyrfc3339': 'types-pyRFC3339',
-    'python2': 'types-six',
-    'pytz': 'types-pytz',
-    'pyVmomi': 'types-pyvmomi',
-    'redis': 'types-redis',
-    'requests': 'types-requests',
-    'retry': 'types-retry',
-    'routes': 'types-Routes',
-    'scribe': 'types-scribe',
-    'simplejson': 'types-simplejson',
-    'singledispatch': 'types-singledispatch',
-    'six': 'types-six',
-    'slugify': 'types-python-slugify',
-    'tabulate': 'types-tabulate',
-    'termcolor': 'types-termcolor',
-    'toml': 'types-toml',
-    'tornado': 'types-tornado',
-    'typed_ast': 'types-typed-ast',
-    'tzlocal': 'types-tzlocal',
-    'ujson': 'types-ujson',
-    'waitress': 'types-waitress',
-    'werkzeug': 'types-Werkzeug',
-    'yaml': 'types-PyYAML',
+    'aiofiles': StubInfo('types-aiofiles', py_version=3),
+    'atomicwrites': StubInfo('types-atomicwrites'),
+    'attr': StubInfo('types-attrs'),
+    'backports': StubInfo('types-backports'),
+    'backports_abc': StubInfo('types-backports_abc'),
+    'bleach': StubInfo('types-bleach'),
+    'boto': StubInfo('types-boto'),
+    'cachetools': StubInfo('types-cachetools'),
+    'certifi': StubInfo('types-certifi'),
+    'characteristic': StubInfo('types-characteristic'),
+    'chardet': StubInfo('types-chardet'),
+    'click': StubInfo('types-click'),
+    'click_spinner': StubInfo('types-click-spinner'),
+    'concurrent': StubInfo('types-futures', py_version=2),
+    'contextvars': StubInfo('types-contextvars', py_version=3),
+    'croniter': StubInfo('types-croniter'),
+    'cryptography': StubInfo('types-cryptography'),
+    'dataclasses': StubInfo('types-dataclasses', py_version=3),
+    'dateparser': StubInfo('types-dateparser'),
+    'datetimerange': StubInfo('types-DateTimeRange'),
+    'dateutil': StubInfo('types-python-dateutil'),
+    'decorator': StubInfo('types-decorator'),
+    'deprecated': StubInfo('types-Deprecated'),
+    'docutils': StubInfo('types-docutils', py_version=3),
+    'emoji': StubInfo('types-emoji'),
+    'enum': StubInfo('types-enum34', py_version=2),
+    'fb303': StubInfo('types-fb303', py_version=2),
+    'filelock': StubInfo('types-filelock', py_version=3),
+    'first': StubInfo('types-first'),
+    'flask': StubInfo('types-Flask'),
+    'freezegun': StubInfo('types-freezegun', py_version=3),
+    'frozendict': StubInfo('types-frozendict', py_version=3),
+    'geoip2': StubInfo('types-geoip2'),
+    'gflags': StubInfo('types-python-gflags'),
+    'google.protobuf': StubInfo('types-protobuf'),
+    'ipaddress': StubInfo('types-ipaddress', py_version=2),
+    'itsdangerous': StubInfo('types-itsdangerous'),
+    'jinja2': StubInfo('types-Jinja2'),
+    'jwt': StubInfo('types-jwt'),
+    'kazoo': StubInfo('types-kazoo', py_version=2),
+    'markdown': StubInfo('types-Markdown'),
+    'markupsafe': StubInfo('types-MarkupSafe'),
+    'maxminddb': StubInfo('types-maxminddb'),
+    'mock': StubInfo('types-mock'),
+    'OpenSSL': StubInfo('types-openssl-python', py_version=2),
+    'orjson': StubInfo('types-orjson', py_version=3),
+    'paramiko': StubInfo('types-paramiko'),
+    'pathlib2': StubInfo('types-pathlib2', py_version=2),
+    'pkg_resources': StubInfo('types-pkg_resources', py_version=3),
+    'polib': StubInfo('types-polib'),
+    'pycurl': StubInfo('types-pycurl'),
+    'pymssql': StubInfo('types-pymssql', py_version=2),
+    'pymysql': StubInfo('types-PyMySQL'),
+    'pyrfc3339': StubInfo('types-pyRFC3339', py_version=3),
+    'python2': StubInfo('types-six'),
+    'pytz': StubInfo('types-pytz'),
+    'pyVmomi': StubInfo('types-pyvmomi'),
+    'redis': StubInfo('types-redis'),
+    'requests': StubInfo('types-requests'),
+    'retry': StubInfo('types-retry'),
+    'routes': StubInfo('types-Routes', py_version=2),
+    'scribe': StubInfo('types-scribe', py_version=2),
+    'simplejson': StubInfo('types-simplejson'),
+    'singledispatch': StubInfo('types-singledispatch'),
+    'six': StubInfo('types-six'),
+    'slugify': StubInfo('types-python-slugify'),
+    'tabulate': StubInfo('types-tabulate'),
+    'termcolor': StubInfo('types-termcolor'),
+    'toml': StubInfo('types-toml'),
+    'tornado': StubInfo('types-tornado', py_version=2),
+    'typed_ast': StubInfo('types-typed-ast', py_version=3),
+    'tzlocal': StubInfo('types-tzlocal'),
+    'ujson': StubInfo('types-ujson'),
+    'waitress': StubInfo('types-waitress', py_version=3),
+    'werkzeug': StubInfo('types-Werkzeug'),
+    'yaml': StubInfo('types-PyYAML'),
 }

mypy/test/testpythoneval.py

@@ -72,6 +72,10 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None
         interpreter = python3_path
         mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))
 
+    m = re.search('# flags: (.*)$', '\n'.join(testcase.input), re.MULTILINE)
+    if m:
+        mypy_cmdline.extend(m.group(1).split())
+
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
     program_path = os.path.join(test_temp_dir, program)

mypy/test/teststubinfo.py

@@ -0,0 +1,18 @@
+import unittest
+
+from mypy.stubinfo import is_legacy_bundled_package
+
+
+class TestStubInfo(unittest.TestCase):
+    def test_is_legacy_bundled_packages(self) -> None:
+        assert not is_legacy_bundled_package('foobar_asdf', 2)
+        assert not is_legacy_bundled_package('foobar_asdf', 3)
+
+        assert is_legacy_bundled_package('click', 2)
+        assert is_legacy_bundled_package('click', 3)
+
+        assert is_legacy_bundled_package('scribe', 2)
+        assert not is_legacy_bundled_package('scribe', 3)
+
+        assert not is_legacy_bundled_package('dataclasses', 2)
+        assert is_legacy_bundled_package('dataclasses', 3)

test-data/unit/python2eval.test

@@ -433,3 +433,17 @@ _testDefaultDictInference.py:5: note: Revealed type is "collections.defaultdict[
 from collections import abc
 [out]
 _testIgnorePython3StdlibStubs_python2.py:1: error: Module "collections" has no attribute "abc"
+
+[case testNoApprovedPython2StubInstalled_python2]
+# flags: --ignore-missing-imports
+import scribe
+from scribe import x
+import maxminddb
+import foobar_asdf
+[out]
+_testNoApprovedPython2StubInstalled_python2.py:2: error: Library stubs not installed for "scribe" (or incompatible with Python 2.7)
+_testNoApprovedPython2StubInstalled_python2.py:2: note: Hint: "python3 -m pip install types-scribe"
+_testNoApprovedPython2StubInstalled_python2.py:2: note: (or run "mypy --install-types" to install all missing stub packages)
+_testNoApprovedPython2StubInstalled_python2.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+_testNoApprovedPython2StubInstalled_python2.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 2.7)
+_testNoApprovedPython2StubInstalled_python2.py:4: note: Hint: "python3 -m pip install types-maxminddb"

test-data/unit/pythoneval.test

@@ -1508,3 +1508,26 @@ x = 0
 [out]
 mypy: "tmp/typing.py" shadows library module "typing"
 note: A user-defined top-level module with name "typing" is not supported
+
+[case testIgnoreImportIfNoPython3StubAvailable]
+# flags: --ignore-missing-imports
+import scribe  # No Python 3 stubs available for scribe
+from scribe import x
+import maxminddb  # Python 3 stubs available for maxminddb
+import foobar_asdf
+[out]
+_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-maxminddb"
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: (or run "mypy --install-types" to install all missing stub packages)
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testNoPython3StubAvailable]
+import scribe
+from scribe import x
+import maxminddb
+[out]
+_testNoPython3StubAvailable.py:1: error: Cannot find implementation or library stub for module named "scribe"
+_testNoPython3StubAvailable.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
+_testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-maxminddb"
+_testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages)