Restrict the number of errors shown when there are missing stubs

docs/source/command_line.rst

@@ -687,6 +687,14 @@ in error messages.
 
     Show absolute paths to files.
 
+.. option:: --soft-error-limit N
+
+    This flag will adjust the limit after which mypy will (sometimes)
+    disable reporting most additional errors. The limit only applies
+    if it seems likely that most of the remaining errors will not be
+    useful or they may be overly noisy. If ``N`` is negative, there is
+    no limit. The default limit is 200.
+
 
 .. _incremental:
 

mypy/build.py

@@ -224,7 +224,8 @@ def _build(sources: List[BuildSource],
                     lambda path: read_py_file(path, cached_read, options.python_version),
                     options.show_absolute_path,
                     options.enabled_error_codes,
-                    options.disabled_error_codes)
+                    options.disabled_error_codes,
+                    options.many_errors_threshold)
     plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)
 
     # Add catch-all .gitignore to cache dir if we created it

mypy/defaults.py

@@ -30,3 +30,7 @@
                   'html',
                   'txt',
                   'lineprecision']  # type: Final
+
+# Threshold after which we sometimes filter out most errors to avoid very
+# verbose output
+MANY_ERRORS_THRESHOLD = 200  # type: Final

mypy/errors.py

@@ -10,7 +10,7 @@
 from mypy.scope import Scope
 from mypy.options import Options
 from mypy.version import __version__ as mypy_version
-from mypy.errorcodes import ErrorCode
+from mypy.errorcodes import ErrorCode, IMPORT
 from mypy import errorcodes as codes
 from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file
 
@@ -65,6 +65,10 @@ class ErrorInfo:
     # Fine-grained incremental target where this was reported
     target = None  # type: Optional[str]
 
+    # If True, don't show this message in output, but still record the error (needed
+    # by mypy daemon)
+    hidden = False
+
     def __init__(self,
                  import_ctx: List[Tuple[str, int]],
                  file: str,
@@ -158,6 +162,10 @@ class Errors:
     target_module = None  # type: Optional[str]
     scope = None  # type: Optional[Scope]
 
+    # Have we seen an import-related error so far? If yes, we filter out other messages
+    # in some cases to avoid reporting huge numbers of errors.
+    seen_import_error = False
+
     def __init__(self,
                  show_error_context: bool = False,
                  show_column_numbers: bool = False,
@@ -166,7 +174,8 @@ def __init__(self,
                  read_source: Optional[Callable[[str], Optional[List[str]]]] = None,
                  show_absolute_path: bool = False,
                  enabled_error_codes: Optional[Set[ErrorCode]] = None,
-                 disabled_error_codes: Optional[Set[ErrorCode]] = None) -> None:
+                 disabled_error_codes: Optional[Set[ErrorCode]] = None,
+                 many_errors_threshold: int = -1) -> None:
         self.show_error_context = show_error_context
         self.show_column_numbers = show_column_numbers
         self.show_error_codes = show_error_codes
@@ -176,6 +185,7 @@ def __init__(self,
         self.read_source = read_source
         self.enabled_error_codes = enabled_error_codes or set()
         self.disabled_error_codes = disabled_error_codes or set()
+        self.many_errors_threshold = many_errors_threshold
         self.initialize()
 
     def initialize(self) -> None:
@@ -189,6 +199,7 @@ def initialize(self) -> None:
         self.only_once_messages = set()
         self.scope = None
         self.target_module = None
+        self.seen_import_error = False
 
     def reset(self) -> None:
         self.initialize()
@@ -201,12 +212,14 @@ def copy(self) -> 'Errors':
                      self.read_source,
                      self.show_absolute_path,
                      self.enabled_error_codes,
-                     self.disabled_error_codes)
+                     self.disabled_error_codes,
+                     self.many_errors_threshold)
         new.file = self.file
         new.import_ctx = self.import_ctx[:]
         new.function_or_member = self.function_or_member[:]
         new.target_module = self.target_module
         new.scope = self.scope
+        new.seen_import_error = self.seen_import_error
         return new
 
     def total_errors(self) -> int:
@@ -330,6 +343,8 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None:
         if file not in self.error_info_map:
             self.error_info_map[file] = []
         self.error_info_map[file].append(info)
+        if info.code is IMPORT:
+            self.seen_import_error = True
 
     def add_error_info(self, info: ErrorInfo) -> None:
         file, line, end_line = info.origin
@@ -354,8 +369,52 @@ def add_error_info(self, info: ErrorInfo) -> None:
             if info.message in self.only_once_messages:
                 return
             self.only_once_messages.add(info.message)
+        if self.seen_import_error and info.code is not IMPORT and self.has_many_errors():
+            # Missing stubs can easily cause thousands of errors about
+            # Any types, especially when upgrading to mypy 0.900,
+            # which no longer bundles third-party library stubs. Avoid
+            # showing too many errors to make it easier to see
+            # import-related errors.
+            info.hidden = True
+            self.report_hidden_errors(info)
         self._add_error_info(file, info)
 
+    def has_many_errors(self) -> bool:
+        if self.many_errors_threshold < 0:
+            return False
+        if len(self.error_info_map) >= self.many_errors_threshold:
+            return True
+        if sum(len(errors)
+               for errors in self.error_info_map.values()) >= self.many_errors_threshold:
+            return True
+        return False
+
+    def report_hidden_errors(self, info: ErrorInfo) -> None:
+        message = (
+            '(Skipping most remaining errors due to unresolved imports or missing stubs; ' +
+            'fix these first)'
+        )
+        if message in self.only_once_messages:
+            return
+        self.only_once_messages.add(message)
+        new_info = ErrorInfo(
+            import_ctx=info.import_ctx,
+            file=info.file,
+            module=info.module,
+            typ=None,
+            function_or_member=None,
+            line=info.line,
+            column=info.line,
+            severity='note',
+            message=message,
+            code=None,
+            blocker=False,
+            only_once=True,
+            origin=info.origin,
+            target=info.target,
+        )
+        self._add_error_info(info.origin[0], new_info)
+
     def is_ignored_error(self, line: int, info: ErrorInfo, ignores: Dict[int, List[str]]) -> bool:
         if info.blocker:
             # Blocking errors can never be ignored
@@ -453,6 +512,7 @@ def format_messages(self, error_info: List[ErrorInfo],
         severity 'error').
         """
         a = []  # type: List[str]
+        error_info = [info for info in error_info if not info.hidden]
         errors = self.render_messages(self.sort_messages(error_info))
         errors = self.remove_duplicates(errors)
         for file, line, column, severity, message, code in errors:

mypy/main.py

@@ -663,6 +663,8 @@ def add_invertible_flag(flag: str,
     add_invertible_flag('--show-absolute-path', default=False,
                         help="Show absolute paths to files",
                         group=error_group)
+    error_group.add_argument('--soft-error-limit', default=defaults.MANY_ERRORS_THRESHOLD,
+                             type=int, dest="many_errors_threshold", help=argparse.SUPPRESS)
 
     incremental_group = parser.add_argument_group(
         title='Incremental mode',

mypy/options.py

@@ -295,6 +295,10 @@ def __init__(self) -> None:
         self.show_absolute_path = False  # type: bool
         # Install missing stub packages if True
         self.install_types = False
+        # When we encounter errors that may cause many additional errors,
+        # skip most errors after this many messages have been reported.
+        # -1 means unlimited.
+        self.many_errors_threshold = defaults.MANY_ERRORS_THRESHOLD
 
     # To avoid breaking plugin compatibility, keep providing new_semantic_analyzer
     @property

test-data/unit/check-modules.test

@@ -2982,3 +2982,85 @@ T = TypeVar("T")
 class F(M):
     x: C
     class C: ...
+
+[case testLimitLegacyStubErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=5
+import certifi # E: Cannot find implementation or library stub for module named "certifi" \
+               # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # N: (Skipping most remaining errors due to unresolved imports or missing stubs; fix these first)
+certifi.x
+certifi.x
+certifi.x
+certifi.x
+
+[case testDoNotLimitErrorVolumeIfNotImportErrors]
+# flags: --disallow-any-expr --soft-error-limit=5
+def f(): pass
+certifi = f()  # E: Expression has type "Any"
+1()  # E: "int" not callable
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+1()  # E: "int" not callable
+
+
+[case testDoNotLimitImportErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=3
+import xyz1  # E: Cannot find implementation or library stub for module named "xyz1" \
+             # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+import xyz2  # E: Cannot find implementation or library stub for module named "xyz2"
+import xyz3  # E: Cannot find implementation or library stub for module named "xyz3"
+import xyz4  # E: Cannot find implementation or library stub for module named "xyz4"
+
+[case testUnlimitedStubErrorVolume]
+# flags: --disallow-any-expr --soft-error-limit=-1
+import certifi # E: Cannot find implementation or library stub for module named "certifi" \
+               # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"
+certifi.x  # E: Expression has type "Any"