From f4272908aee8ea4e2b1433221bceb5b893b2a91c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 13:30:22 +0100 Subject: [PATCH 1/8] Run build a second time when using --install-types --non-interactive If the first build finds missing stub packages, run the build a second time after installing types. Only show errors from the last build. Work on #10600. --- mypy/main.py | 124 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 45 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 735d9c0778e8..def0a0ff7531 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -44,6 +44,55 @@ def stat_proxy(path: str) -> os.stat_result: return st +def run_build(sources: List[BuildSource], + options: Options, + fscache: FileSystemCache, + t0: float, + stdout: TextIO, + stderr: TextIO) -> Tuple[Optional[build.BuildResult], List[str], bool]: + formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) + + messages = [] + + def flush_errors(new_messages: List[str], serious: bool) -> None: + if options.pretty: + new_messages = formatter.fit_in_terminal(new_messages) + messages.extend(new_messages) + if options.non_interactive: + # Collect messages and possibly show them later. + return + f = stderr if serious else stdout + for msg in new_messages: + if options.color_output: + msg = formatter.colorize(msg) + f.write(msg + '\n') + f.flush() + + serious = False + blockers = False + res = None + try: + # Keep a dummy reference (res) for memory profiling below, as otherwise + # the result could be freed. + res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) + except CompileError as e: + blockers = True + if not e.use_stdout: + serious = True + if (options.warn_unused_configs + and options.unused_configs + and not options.incremental + and not options.non_interactive): + print("Warning: unused section(s) in %s: %s" % + (options.config_file, + get_config_module_names(options.config_file, + [glob for glob in options.per_module_options.keys() + if glob in options.unused_configs])), + file=stderr) + maybe_write_junit_xml(time.time() - t0, serious, messages, options) + return res, messages, blockers + + def main(script_path: Optional[str], stdout: TextIO, stderr: TextIO, @@ -67,7 +116,6 @@ def main(script_path: Optional[str], sources, options = process_options(args, stdout=stdout, stderr=stderr, fscache=fscache) - messages = [] formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): @@ -85,41 +133,18 @@ def main(script_path: Optional[str], install_types(options.cache_dir, formatter, non_interactive=options.non_interactive) return - def flush_errors(new_messages: List[str], serious: bool) -> None: - if options.non_interactive: - return - if options.pretty: - new_messages = formatter.fit_in_terminal(new_messages) - messages.extend(new_messages) - f = stderr if serious else stdout - for msg in new_messages: - if options.color_output: - msg = formatter.colorize(msg) - f.write(msg + '\n') - f.flush() + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) - serious = False - blockers = False - res = None - try: - # Keep a dummy reference (res) for memory profiling below, as otherwise - # the result could be freed. - res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) - except CompileError as e: - blockers = True - if not e.use_stdout: - serious = True - if (options.warn_unused_configs - and options.unused_configs - and not options.incremental - and not options.non_interactive): - print("Warning: unused section(s) in %s: %s" % - (options.config_file, - get_config_module_names(options.config_file, - [glob for glob in options.per_module_options.keys() - if glob in options.unused_configs])), - file=stderr) - maybe_write_junit_xml(time.time() - t0, serious, messages, options) + if options.non_interactive: + missing_pkgs = read_types_packages_to_install(options.cache_dir, after_run=True) + if missing_pkgs: + # Install missing type packages and rerun build. + install_types(options.cache_dir, formatter, after_run=True, non_interactive=True) + fscache.flush() + print() + res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) + for msg in messages: + stderr.write(msg + '\n') if MEM_PROFILE: from mypy.memprofile import print_memory_profile @@ -128,7 +153,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: code = 0 if messages: code = 2 if blockers else 1 - if options.error_summary and not options.non_interactive: + if options.error_summary: if messages: n_errors, n_files = util.count_stats(messages) if n_errors: @@ -141,9 +166,11 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: stdout.write(formatter.format_success(len(sources), options.color_output) + '\n') stdout.flush() - if options.install_types: + if options.install_types and not options.non_interactive: install_types(options.cache_dir, formatter, after_run=True, non_interactive=options.non_interactive) + print() + print("Hint: Run mypy again to get up-to-date results with installed types") return if options.fast_exit: @@ -1087,12 +1114,7 @@ def fail(msg: str, stderr: TextIO, options: Options) -> None: sys.exit(2) -def install_types(cache_dir: str, - formatter: util.FancyFormatter, - *, - after_run: bool = False, - non_interactive: bool = False) -> None: - """Install stub packages using pip if some missing stubs were detected.""" +def read_types_packages_to_install(cache_dir: str, after_run: bool) -> List[str]: if not os.path.isdir(cache_dir): if not after_run: sys.stderr.write( @@ -1106,10 +1128,22 @@ def install_types(cache_dir: str, sys.exit(2) fnam = build.missing_stubs_file(cache_dir) if not os.path.isfile(fnam): + # No missing stubs. + return [] + with open(fnam) as f: + return [line.strip() for line in f.readlines()] + + +def install_types(cache_dir: str, + formatter: util.FancyFormatter, + *, + after_run: bool = False, + non_interactive: bool = False) -> None: + """Install stub packages using pip if some missing stubs were detected.""" + packages = read_types_packages_to_install(cache_dir, after_run) + if not packages: # If there are no missing stubs, generate no output. return - with open(fnam) as f: - packages = [line.strip() for line in f.readlines()] if after_run and not non_interactive: print() print('Installing missing stub packages:') From 2fb05b1a35d6c790515f3df854e1cde39f0004f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 13:53:21 +0100 Subject: [PATCH 2/8] Refactor and colorize output --- mypy/main.py | 108 +++++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index def0a0ff7531..4e3eaf7f8bb7 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -44,55 +44,6 @@ def stat_proxy(path: str) -> os.stat_result: return st -def run_build(sources: List[BuildSource], - options: Options, - fscache: FileSystemCache, - t0: float, - stdout: TextIO, - stderr: TextIO) -> Tuple[Optional[build.BuildResult], List[str], bool]: - formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) - - messages = [] - - def flush_errors(new_messages: List[str], serious: bool) -> None: - if options.pretty: - new_messages = formatter.fit_in_terminal(new_messages) - messages.extend(new_messages) - if options.non_interactive: - # Collect messages and possibly show them later. - return - f = stderr if serious else stdout - for msg in new_messages: - if options.color_output: - msg = formatter.colorize(msg) - f.write(msg + '\n') - f.flush() - - serious = False - blockers = False - res = None - try: - # Keep a dummy reference (res) for memory profiling below, as otherwise - # the result could be freed. - res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) - except CompileError as e: - blockers = True - if not e.use_stdout: - serious = True - if (options.warn_unused_configs - and options.unused_configs - and not options.incremental - and not options.non_interactive): - print("Warning: unused section(s) in %s: %s" % - (options.config_file, - get_config_module_names(options.config_file, - [glob for glob in options.per_module_options.keys() - if glob in options.unused_configs])), - file=stderr) - maybe_write_junit_xml(time.time() - t0, serious, messages, options) - return res, messages, blockers - - def main(script_path: Optional[str], stdout: TextIO, stderr: TextIO, @@ -143,8 +94,7 @@ def main(script_path: Optional[str], fscache.flush() print() res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr) - for msg in messages: - stderr.write(msg + '\n') + show_messages(messages, stderr, formatter, options) if MEM_PROFILE: from mypy.memprofile import print_memory_profile @@ -185,6 +135,62 @@ def main(script_path: Optional[str], list([res]) +def run_build(sources: List[BuildSource], + options: Options, + fscache: FileSystemCache, + t0: float, + stdout: TextIO, + stderr: TextIO) -> Tuple[Optional[build.BuildResult], List[str], bool]: + formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) + + messages = [] + + def flush_errors(new_messages: List[str], serious: bool) -> None: + if options.pretty: + new_messages = formatter.fit_in_terminal(new_messages) + messages.extend(new_messages) + if options.non_interactive: + # Collect messages and possibly show them later. + return + f = stderr if serious else stdout + show_messages(messages, f, formatter, options) + + serious = False + blockers = False + res = None + try: + # Keep a dummy reference (res) for memory profiling below, as otherwise + # the result could be freed. + res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) + except CompileError as e: + blockers = True + if not e.use_stdout: + serious = True + if (options.warn_unused_configs + and options.unused_configs + and not options.incremental + and not options.non_interactive): + print("Warning: unused section(s) in %s: %s" % + (options.config_file, + get_config_module_names(options.config_file, + [glob for glob in options.per_module_options.keys() + if glob in options.unused_configs])), + file=stderr) + maybe_write_junit_xml(time.time() - t0, serious, messages, options) + return res, messages, blockers + + +def show_messages(messages: List[str], + f: TextIO, + formatter: util.FancyFormatter, + options: Options) -> None: + for msg in messages: + if options.color_output: + msg = formatter.colorize(msg) + f.write(msg + '\n') + f.flush() + + # Make the help output a little less jarring. class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog: str) -> None: From afced550ff09f1f6016c4173760c252926b2f483 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 14:21:20 +0100 Subject: [PATCH 3/8] Fix test and add another test --- test-data/unit/cmdline.test | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index e469c48a48bf..9e467e30d3ee 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1288,3 +1288,10 @@ Error: --non-interactive is only supported with --install-types [file pkg.py] 1() [out] +pkg.py:1: error: "int" not callable + +[case testCmdlineNonInteractiveInstallTypesNothingToDoNoError] +# cmd: mypy --install-types --non-interactive -m pkg +[file pkg.py] +1 + 2 +[out] From 694fa7ed33b4baf3e2aee3f8c56b558c3c1b46af Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 14:21:30 +0100 Subject: [PATCH 4/8] Fix duplicate messages --- mypy/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 4e3eaf7f8bb7..2ff61ed0f338 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -153,13 +153,13 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: # Collect messages and possibly show them later. return f = stderr if serious else stdout - show_messages(messages, f, formatter, options) + show_messages(new_messages, f, formatter, options) serious = False blockers = False res = None try: - # Keep a dummy reference (res) for memory profiling below, as otherwise + # Keep a dummy reference (res) for memory profiling afterwards, as otherwise # the result could be freed. res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) except CompileError as e: From faaab78d15f2f93473912209fdd89a7519b7abb9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 14:32:25 +0100 Subject: [PATCH 5/8] Update message --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 2ff61ed0f338..fe0cb8b0d8bf 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -120,7 +120,7 @@ def main(script_path: Optional[str], install_types(options.cache_dir, formatter, after_run=True, non_interactive=options.non_interactive) print() - print("Hint: Run mypy again to get up-to-date results with installed types") + print("Hint: Run mypy again for up-to-date results with installed types") return if options.fast_exit: From 8fcaf9100013f0a45d8683e3fa3764f5ed816705 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 15:29:33 +0100 Subject: [PATCH 6/8] Fix exit code and trim output when nothing installed --- mypy/main.py | 16 +++++++++------- test-data/unit/cmdline.test | 7 +++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index fe0cb8b0d8bf..535b055b3e13 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -117,11 +117,12 @@ def main(script_path: Optional[str], stdout.flush() if options.install_types and not options.non_interactive: - install_types(options.cache_dir, formatter, after_run=True, - non_interactive=options.non_interactive) - print() - print("Hint: Run mypy again for up-to-date results with installed types") - return + result = install_types(options.cache_dir, formatter, after_run=True, + non_interactive=options.non_interactive) + if result: + print() + print("Hint: Run mypy again for up-to-date results with installed types") + return if options.fast_exit: # Exit without freeing objects -- it's faster. @@ -1144,12 +1145,12 @@ def install_types(cache_dir: str, formatter: util.FancyFormatter, *, after_run: bool = False, - non_interactive: bool = False) -> None: + non_interactive: bool = False) -> bool: """Install stub packages using pip if some missing stubs were detected.""" packages = read_types_packages_to_install(cache_dir, after_run) if not packages: # If there are no missing stubs, generate no output. - return + return False if after_run and not non_interactive: print() print('Installing missing stub packages:') @@ -1163,3 +1164,4 @@ def install_types(cache_dir: str, sys.exit(2) print() subprocess.run(cmd) + return True diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 9e467e30d3ee..1c14981d5697 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1295,3 +1295,10 @@ pkg.py:1: error: "int" not callable [file pkg.py] 1 + 2 [out] + +[case testCmdlineInteractiveInstallTypesNothingToDo] +# cmd: mypy --install-types -m pkg +[file pkg.py] +1() +[out] +pkg.py:1: error: "int" not callable From 74eccea91186d00b9114f9212e0918848fb76c9d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 15:33:13 +0100 Subject: [PATCH 7/8] Tweak error messages and exit code --- mypy/main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 535b055b3e13..103da54dcd74 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -71,13 +71,13 @@ def main(script_path: Optional[str], if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. - fail("Error: --install-types not supported in this mode of running mypy", stderr, options) + fail("error: --install-types not supported in this mode of running mypy", stderr, options) if options.non_interactive and not options.install_types: - fail("Error: --non-interactive is only supported with --install-types", stderr, options) + fail("error: --non-interactive is only supported with --install-types", stderr, options) if options.install_types and not options.incremental: - fail("Error: --install-types not supported with incremental mode disabled", + fail("error: --install-types not supported with incremental mode disabled", stderr, options) if options.install_types and not sources: @@ -121,8 +121,8 @@ def main(script_path: Optional[str], non_interactive=options.non_interactive) if result: print() - print("Hint: Run mypy again for up-to-date results with installed types") - return + print("note: Run mypy again for up-to-date results with installed types") + code = 2 if options.fast_exit: # Exit without freeing objects -- it's faster. @@ -1125,12 +1125,12 @@ def read_types_packages_to_install(cache_dir: str, after_run: bool) -> List[str] if not os.path.isdir(cache_dir): if not after_run: sys.stderr.write( - "Error: Can't determine which types to install with no files to check " + + "error: Can't determine which types to install with no files to check " + "(and no cache from previous mypy run)\n" ) else: sys.stderr.write( - "Error: --install-types failed (no mypy cache directory)\n" + "error: --install-types failed (no mypy cache directory)\n" ) sys.exit(2) fnam = build.missing_stubs_file(cache_dir) From 388b55e7b5e3bb84bfa3e645f8eb49de800eb8e5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Jun 2021 16:02:30 +0100 Subject: [PATCH 8/8] Fix test --- mypy/main.py | 2 +- test-data/unit/cmdline.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 103da54dcd74..da4eda6e04a0 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -118,7 +118,7 @@ def main(script_path: Optional[str], if options.install_types and not options.non_interactive: result = install_types(options.cache_dir, formatter, after_run=True, - non_interactive=options.non_interactive) + non_interactive=False) if result: print() print("note: Run mypy again for up-to-date results with installed types") diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 1c14981d5697..92ef7e0690ed 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1280,7 +1280,7 @@ pkg.py:1: error: Incompatible types in assignment (expression has type "int", va [case testCmdlineNonInteractiveWithoutInstallTypes] # cmd: mypy --non-interactive -m pkg [out] -Error: --non-interactive is only supported with --install-types +error: --non-interactive is only supported with --install-types == Return code: 2 [case testCmdlineNonInteractiveInstallTypesNothingToDo]