main.py

"""Mypy type checker command line tool."""

import argparse
import os
import subprocess
import sys
import time

from typing import Any, Dict, List, Optional, Tuple, TextIO
from typing_extensions import Final

from mypy import build
from mypy import defaults
from mypy import state
from mypy import util
from mypy.modulefinder import BuildSource, FindModuleCache, mypy_path, SearchPaths
from mypy.find_sources import create_source_list, InvalidSourceList
from mypy.fscache import FileSystemCache
from mypy.errors import CompileError
from mypy.options import Options, BuildType
from mypy.config_parser import parse_version, parse_config_file
from mypy.split_namespace import SplitNamespace

from mypy.version import __version__

orig_stat = os.stat  # type: Final
MEM_PROFILE = False  # type: Final  # If True, dump memory profile


def stat_proxy(path: str) -> os.stat_result:
    try:
        st = orig_stat(path)
    except os.error as err:
        print("stat(%r) -> %s" % (path, err))
        raise
    else:
        print("stat(%r) -> (st_mode=%o, st_mtime=%d, st_size=%d)" %
              (path, st.st_mode, st.st_mtime, st.st_size))
        return st


def main(script_path: Optional[str],
         stdout: TextIO,
         stderr: TextIO,
         args: Optional[List[str]] = None,
         ) -> None:
    """Main entry point to the type checker.

    Args:
        script_path: Path to the 'mypy' script (used for finding data files).
        args: Custom command-line arguments.  If not given, sys.argv[1:] will
        be used.
    """
    util.check_python_version('mypy')
    t0 = time.time()
    # To log stat() calls: os.stat = stat_proxy
    sys.setrecursionlimit(2 ** 14)
    if args is None:
        args = sys.argv[1:]

    fscache = FileSystemCache()
    sources, options = process_options(args, stdout=stdout, stderr=stderr,
                                       fscache=fscache)

    messages = []
    formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes)

    def flush_errors(new_messages: List[str], serious: bool) -> None:
        if options.pretty:
            new_messages = formatter.fit_in_terminal(new_messages)
        messages.extend(new_messages)
        f = stderr if serious else stdout
        try:
            for msg in new_messages:
                if options.color_output:
                    msg = formatter.colorize(msg)
                f.write(msg + '\n')
            f.flush()
        except BrokenPipeError:
            sys.exit(2)

    serious = False
    blockers = False
    res = None
    try:
        # Keep a dummy reference (res) for memory profiling below, as otherwise
        # the result could be freed.
        res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr)
    except CompileError as e:
        blockers = True
        if not e.use_stdout:
            serious = True
    if options.warn_unused_configs and options.unused_configs and not options.incremental:
        print("Warning: unused section(s) in %s: %s" %
              (options.config_file,
               ", ".join("[mypy-%s]" % glob for glob in options.per_module_options.keys()
                         if glob in options.unused_configs)),
              file=stderr)
    if options.junit_xml:
        t1 = time.time()
        py_version = '{}_{}'.format(options.python_version[0], options.python_version[1])
        util.write_junit_xml(t1 - t0, serious, messages, options.junit_xml,
                             py_version, options.platform)

    if MEM_PROFILE:
        from mypy.memprofile import print_memory_profile
        print_memory_profile()
    del res  # Now it's safe to delete

    code = 0
    if messages:
        code = 2 if blockers else 1
    if options.error_summary:
        if messages:
            n_errors, n_files = util.count_stats(messages)
            if n_errors:
                stdout.write(formatter.format_error(n_errors, n_files, len(sources),
                                                    options.color_output) + '\n')
        else:
            stdout.write(formatter.format_success(len(sources),
                                                  options.color_output) + '\n')
        stdout.flush()
    if options.fast_exit:
        # Exit without freeing objects -- it's faster.
        #
        # NOTE: We don't flush all open files on exit (or run other destructors)!
        util.hard_exit(code)
    elif code:
        sys.exit(code)


# Make the help output a little less jarring.
class AugmentedHelpFormatter(argparse.RawDescriptionHelpFormatter):
    def __init__(self, prog: str) -> None:
        super().__init__(prog=prog, max_help_position=28)

    # FIXME: typeshed incorrectly has the type of indent as int when
    # it should be str. Make it Any to avoid rusing mypyc.
    def _fill_text(self, text: str, width: int, indent: Any) -> str:
        if '\n' in text:
            # Assume we want to manually format the text
            return super()._fill_text(text, width, indent)
        else:
            # Assume we want argparse to manage wrapping, indentating, and
            # formatting the text for us.
            return argparse.HelpFormatter._fill_text(self, text, width, indent)


# Define pairs of flag prefixes with inverse meaning.
flag_prefix_pairs = [
    ('allow', 'disallow'),
    ('show', 'hide'),
]  # type: Final
flag_prefix_map = {}  # type: Final[Dict[str, str]]
for a, b in flag_prefix_pairs:
    flag_prefix_map[a] = b
    flag_prefix_map[b] = a


def invert_flag_name(flag: str) -> str:
    split = flag[2:].split('-', 1)
    if len(split) == 2:
        prefix, rest = split
        if prefix in flag_prefix_map:
            return '--{}-{}'.format(flag_prefix_map[prefix], rest)
        elif prefix == 'no':
            return '--{}'.format(rest)

    return '--no-{}'.format(flag[2:])


class PythonExecutableInferenceError(Exception):
    """Represents a failure to infer the version or executable while searching."""


def python_executable_prefix(v: str) -> List[str]:
    if sys.platform == 'win32':
        # on Windows, all Python executables are named `python`. To handle this, there
        # is the `py` launcher, which can be passed a version e.g. `py -3.5`, and it will
        # execute an installed Python 3.5 interpreter. See also:
        # https://docs.python.org/3/using/windows.html#python-launcher-for-windows
        return ['py', '-{}'.format(v)]
    else:
        return ['python{}'.format(v)]


def _python_executable_from_version(python_version: Tuple[int, int]) -> str:
    if sys.version_info[:2] == python_version:
        return sys.executable
    str_ver = '.'.join(map(str, python_version))
    try:
        sys_exe = subprocess.check_output(python_executable_prefix(str_ver) +
                                          ['-c', 'import sys; print(sys.executable)'],
                                          stderr=subprocess.STDOUT).decode().strip()
        return sys_exe
    except (subprocess.CalledProcessError, FileNotFoundError):
        raise PythonExecutableInferenceError(
            'failed to find a Python executable matching version {},'
            ' perhaps try --python-executable, or --no-site-packages?'.format(python_version))


def infer_python_executable(options: Options,
                            special_opts: argparse.Namespace) -> None:
    """Infer the Python executable from the given version.

    This function mutates options based on special_opts to infer the correct Python executable
    to use.
    """
    # TODO: (ethanhs) Look at folding these checks and the site packages subprocess calls into
    # one subprocess call for speed.

    # Use the command line specified executable, or fall back to one set in the
    # config file. If an executable is not specified, infer it from the version
    # (unless no_executable is set)
    python_executable = special_opts.python_executable or options.python_executable

    if python_executable is None:
        if not special_opts.no_executable:
            python_executable = _python_executable_from_version(options.python_version)
    options.python_executable = python_executable


HEADER = """%(prog)s [-h] [-v] [-V] [more options; see below]
            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]"""  # type: Final


DESCRIPTION = """
Mypy is a program that will type check your Python code.

Pass in any files or folders you want to type check. Mypy will
recursively traverse any provided folders to find .py files:

    $ mypy my_program.py my_src_folder

For more information on getting started, see:

- http://mypy.readthedocs.io/en/latest/getting_started.html

For more details on both running mypy and using the flags below, see:

- http://mypy.readthedocs.io/en/latest/running_mypy.html
- http://mypy.readthedocs.io/en/latest/command_line.html

You can also use a config file to configure mypy instead of using
command line flags. For more details, see:

- http://mypy.readthedocs.io/en/latest/config_file.html
"""  # type: Final

FOOTER = """Environment variables:
  Define MYPYPATH for additional module search path entries.
  Define MYPY_CACHE_DIR to override configuration cache_dir path."""  # type: Final


def process_options(args: List[str],
                    stdout: Optional[TextIO] = None,
                    stderr: Optional[TextIO] = None,
                    require_targets: bool = True,
                    server_options: bool = False,
                    fscache: Optional[FileSystemCache] = None,
                    program: str = 'mypy',
                    header: str = HEADER,
                    ) -> Tuple[List[BuildSource], Options]:
    """Parse command line arguments.

    If a FileSystemCache is passed in, and package_root options are given,
    call fscache.set_package_root() to set the cache's package root.
    """
    stdout = stdout or sys.stdout
    stderr = stderr or sys.stderr

    parser = argparse.ArgumentParser(prog=program,
                                     usage=header,
                                     description=DESCRIPTION,
                                     epilog=FOOTER,
                                     fromfile_prefix_chars='@',
                                     formatter_class=AugmentedHelpFormatter,
                                     add_help=False)

    strict_flag_names = []  # type: List[str]
    strict_flag_assignments = []  # type: List[Tuple[str, bool]]

    def add_invertible_flag(flag: str,
                            *,
                            inverse: Optional[str] = None,
                            default: bool,
                            dest: Optional[str] = None,
                            help: str,
                            strict_flag: bool = False,
                            group: Optional[argparse._ActionsContainer] = None
                            ) -> None:
        if inverse is None:
            inverse = invert_flag_name(flag)
        if group is None:
            group = parser

        if help is not argparse.SUPPRESS:
            help += " (inverse: {})".format(inverse)

        arg = group.add_argument(flag,
                                 action='store_false' if default else 'store_true',
                                 dest=dest,
                                 help=help)
        dest = arg.dest
        arg = group.add_argument(inverse,
                                 action='store_true' if default else 'store_false',
                                 dest=dest,
                                 help=argparse.SUPPRESS)
        if strict_flag:
            assert dest is not None
            strict_flag_names.append(flag)
            strict_flag_assignments.append((dest, not default))

    # Unless otherwise specified, arguments will be parsed directly onto an
    # Options object.  Options that require further processing should have
    # their `dest` prefixed with `special-opts:`, which will cause them to be
    # parsed into the separate special_opts namespace object.

    # Note: we have a style guide for formatting the mypy --help text. See
    # https://github.com/python/mypy/wiki/Documentation-Conventions

    general_group = parser.add_argument_group(
        title='Optional arguments')
    general_group.add_argument(
        '-h', '--help', action='help',
        help="Show this help message and exit")
    general_group.add_argument(
        '-v', '--verbose', action='count', dest='verbosity',
        help="More verbose messages")
    general_group.add_argument(
        '-V', '--version', action='version',
        version='%(prog)s ' + __version__,
        help="Show program's version number and exit")

    config_group = parser.add_argument_group(
        title='Config file',
        description="Use a config file instead of command line arguments. "
                    "This is useful if you are using many flags or want "
                    "to set different options per each module.")
    config_group.add_argument(
        '--config-file',
        help="Configuration file, must have a [mypy] section "
             "(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
    add_invertible_flag('--warn-unused-configs', default=False, strict_flag=True,
                        help="Warn about unused '[mypy-<pattern>]' config sections",
                        group=config_group)

    imports_group = parser.add_argument_group(
        title='Import discovery',
        description="Configure how imports are discovered and followed.")
    imports_group.add_argument(
        '--ignore-missing-imports', action='store_true',
        help="Silently ignore imports of missing modules")
    imports_group.add_argument(
        '--follow-imports', choices=['normal', 'silent', 'skip', 'error'],
        default='normal', help="How to treat imports (default normal)")
    imports_group.add_argument(
        '--python-executable', action='store', metavar='EXECUTABLE',
        help="Python executable used for finding PEP 561 compliant installed"
             " packages and stubs",
        dest='special-opts:python_executable')
    imports_group.add_argument(
        '--no-site-packages', action='store_true',
        dest='special-opts:no_executable',
        help="Do not search for installed PEP 561 compliant packages")
    imports_group.add_argument(
        '--no-silence-site-packages', action='store_true',
        help="Do not silence errors in PEP 561 compliant installed packages")
    add_invertible_flag(
        '--namespace-packages', default=False,
        help="Support namespace packages (PEP 420, __init__.py-less)",
        group=imports_group)

    platform_group = parser.add_argument_group(
        title='Platform configuration',
        description="Type check code assuming it will be run under certain "
                    "runtime conditions. By default, mypy assumes your code "
                    "will be run using the same operating system and Python "
                    "version you are using to run mypy itself.")
    platform_group.add_argument(
        '--python-version', type=parse_version, metavar='x.y',
        help='Type check code assuming it will be running on Python x.y',
        dest='special-opts:python_version')
    platform_group.add_argument(
        '-2', '--py2', dest='special-opts:python_version', action='store_const',
        const=defaults.PYTHON2_VERSION,
        help="Use Python 2 mode (same as --python-version 2.7)")
    platform_group.add_argument(
        '--platform', action='store', metavar='PLATFORM',
        help="Type check special-cased code for the given OS platform "
             "(defaults to sys.platform)")
    platform_group.add_argument(
        '--always-true', metavar='NAME', action='append', default=[],
        help="Additional variable to be considered True (may be repeated)")
    platform_group.add_argument(
        '--always-false', metavar='NAME', action='append', default=[],
        help="Additional variable to be considered False (may be repeated)")

    disallow_any_group = parser.add_argument_group(
        title='Dynamic typing',
        description="Disallow the use of the dynamic 'Any' type under certain conditions.")
    disallow_any_group.add_argument(
        '--disallow-any-unimported', default=False, action='store_true',
        help="Disallow Any types resulting from unfollowed imports")
    add_invertible_flag('--disallow-subclassing-any', default=False, strict_flag=True,
                        help="Disallow subclassing values of type 'Any' when defining classes",
                        group=disallow_any_group)
    disallow_any_group.add_argument(
        '--disallow-any-expr', default=False, action='store_true',
        help='Disallow all expressions that have type Any')
    disallow_any_group.add_argument(
        '--disallow-any-decorated', default=False, action='store_true',
        help='Disallow functions that have Any in their signature '
             'after decorator transformation')
    disallow_any_group.add_argument(
        '--disallow-any-explicit', default=False, action='store_true',
        help='Disallow explicit Any in type positions')
    add_invertible_flag('--disallow-any-generics', default=False, strict_flag=True,
                        help='Disallow usage of generic types that do not specify explicit type '
                        'parameters', group=disallow_any_group)

    untyped_group = parser.add_argument_group(
        title='Untyped definitions and calls',
        description="Configure how untyped definitions and calls are handled. "
                    "Note: by default, mypy ignores any untyped function definitions "
                    "and assumes any calls to such functions have a return "
                    "type of 'Any'.")
    add_invertible_flag('--disallow-untyped-calls', default=False, strict_flag=True,
                        help="Disallow calling functions without type annotations"
                        " from functions with type annotations",
                        group=untyped_group)
    add_invertible_flag('--disallow-untyped-defs', default=False, strict_flag=True,
                        help="Disallow defining functions without type annotations"
                        " or with incomplete type annotations",
                        group=untyped_group)
    add_invertible_flag('--disallow-incomplete-defs', default=False, strict_flag=True,
                        help="Disallow defining functions with incomplete type annotations",
                        group=untyped_group)
    add_invertible_flag('--check-untyped-defs', default=False, strict_flag=True,
                        help="Type check the interior of functions without type annotations",
                        group=untyped_group)
    add_invertible_flag('--disallow-untyped-decorators', default=False, strict_flag=True,
                        help="Disallow decorating typed functions with untyped decorators",
                        group=untyped_group)

    none_group = parser.add_argument_group(
        title='None and Optional handling',
        description="Adjust how values of type 'None' are handled. For more context on "
                    "how mypy handles values of type 'None', see: "
                    "http://mypy.readthedocs.io/en/latest/kinds_of_types.html#no-strict-optional")
    add_invertible_flag('--no-implicit-optional', default=False, strict_flag=True,
                        help="Don't assume arguments with default values of None are Optional",
                        group=none_group)
    none_group.add_argument(
        '--strict-optional', action='store_true',
        help=argparse.SUPPRESS)
    none_group.add_argument(
        '--no-strict-optional', action='store_false', dest='strict_optional',
        help="Disable strict Optional checks (inverse: --strict-optional)")
    none_group.add_argument(
        '--strict-optional-whitelist', metavar='GLOB', nargs='*',
        help="Suppress strict Optional errors in all but the provided files; "
             "implies --strict-optional (may suppress certain other errors "
             "in non-whitelisted files)")

    lint_group = parser.add_argument_group(
        title='Warnings',
        description="Detect code that is sound but redundant or problematic.")
    add_invertible_flag('--warn-redundant-casts', default=False, strict_flag=True,
                        help="Warn about casting an expression to its inferred type",
                        group=lint_group)
    add_invertible_flag('--warn-unused-ignores', default=False, strict_flag=True,
                        help="Warn about unneeded '# type: ignore' comments",
                        group=lint_group)
    add_invertible_flag('--no-warn-no-return', dest='warn_no_return', default=True,
                        help="Do not warn about functions that end without returning",
                        group=lint_group)
    add_invertible_flag('--warn-return-any', default=False, strict_flag=True,
                        help="Warn about returning values of type Any"
                             " from non-Any typed functions",
                        group=lint_group)
    add_invertible_flag('--warn-unreachable', default=False, strict_flag=False,
                        help="Warn about statements or expressions inferred to be"
                             " unreachable or redundant",
                        group=lint_group)

    # Note: this group is intentionally added here even though we don't add
    # --strict to this group near the end.
    #
    # That way, this group will appear after the various strictness groups
    # but before the remaining flags.
    # We add `--strict` near the end so we don't accidentally miss any strictness
    # flags that are added after this group.
    strictness_group = parser.add_argument_group(
        title='Other strictness checks')

    add_invertible_flag('--allow-untyped-globals', default=False, strict_flag=False,
                        help="Suppress toplevel errors caused by missing annotations",
                        group=strictness_group)

    add_invertible_flag('--allow-redefinition', default=False, strict_flag=False,
                        help="Allow unconditional variable redefinition with a new type",
                        group=strictness_group)

    add_invertible_flag('--strict-equality', default=False, strict_flag=False,
                        help="Prohibit equality, identity, and container checks for"
                             " non-overlapping types",
                        group=strictness_group)

    add_invertible_flag('--no-implicit-reexport', default=True, strict_flag=True,
                        dest='implicit_reexport',
                        help="Treat imports as private unless aliased",
                        group=strictness_group)

    incremental_group = parser.add_argument_group(
        title='Incremental mode',
        description="Adjust how mypy incrementally type checks and caches modules. "
                    "Mypy caches type information about modules into a cache to "
                    "let you speed up future invocations of mypy. Also see "
                    "mypy's daemon mode: "
                    "mypy.readthedocs.io/en/latest/mypy_daemon.html#mypy-daemon")
    incremental_group.add_argument(
        '-i', '--incremental', action='store_true',
        help=argparse.SUPPRESS)
    incremental_group.add_argument(
        '--no-incremental', action='store_false', dest='incremental',
        help="Disable module cache (inverse: --incremental)")
    incremental_group.add_argument(
        '--cache-dir', action='store', metavar='DIR',
        help="Store module cache info in the given folder in incremental mode "
             "(defaults to '{}')".format(defaults.CACHE_DIR))
    add_invertible_flag('--sqlite-cache', default=False,
                        help="Use a sqlite database to store the cache",
                        group=incremental_group)
    incremental_group.add_argument(
        '--cache-fine-grained', action='store_true',
        help="Include fine-grained dependency information in the cache for the mypy daemon")
    incremental_group.add_argument(
        '--skip-version-check', action='store_true',
        help="Allow using cache written by older mypy version")
    incremental_group.add_argument(
        '--skip-cache-mtime-checks', action='store_true',
        help="Skip cache internal consistency checks based on mtime")

    internals_group = parser.add_argument_group(
        title='Mypy internals',
        description="Debug and customize mypy internals.")
    internals_group.add_argument(
        '--pdb', action='store_true', help="Invoke pdb on fatal error")
    internals_group.add_argument(
        '--show-traceback', '--tb', action='store_true',
        help="Show traceback on fatal error")
    internals_group.add_argument(
        '--raise-exceptions', action='store_true', help="Raise exception on fatal error"
    )
    internals_group.add_argument(
        '--custom-typing', metavar='MODULE', dest='custom_typing_module',
        help="Use a custom typing module")
    internals_group.add_argument(
        '--custom-typeshed-dir', metavar='DIR',
        help="Use the custom typeshed in DIR")
    add_invertible_flag('--warn-incomplete-stub', default=False,
                        help="Warn if missing type annotation in typeshed, only relevant with"
                             " --disallow-untyped-defs or --disallow-incomplete-defs enabled",
                        group=internals_group)
    internals_group.add_argument(
        '--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
        dest='shadow_file', action='append',
        help="When encountering SOURCE_FILE, read and type check "
             "the contents of SHADOW_FILE instead.")
    add_invertible_flag('--fast-exit', default=False, help=argparse.SUPPRESS,
                        group=internals_group)

    error_group = parser.add_argument_group(
        title='Error reporting',
        description="Adjust the amount of detail shown in error messages.")
    add_invertible_flag('--show-error-context', default=False,
                        dest='show_error_context',
                        help='Precede errors with "note:" messages explaining context',
                        group=error_group)
    add_invertible_flag('--show-column-numbers', default=False,
                        help="Show column numbers in error messages",
                        group=error_group)
    add_invertible_flag('--show-error-codes', default=False,
                        help="Show error codes in error messages",
                        group=error_group)
    add_invertible_flag('--pretty', default=False,
                        help="Use visually nicer output in error messages:"
                             " Use soft word wrap, show source code snippets,"
                             " and error location markers",
                        group=error_group)
    add_invertible_flag('--no-color-output', dest='color_output', default=True,
                        help="Do not colorize error messages",
                        group=error_group)
    add_invertible_flag('--no-error-summary', dest='error_summary', default=True,
                        help="Do not show error stats summary",
                        group=error_group)

    strict_help = "Strict mode; enables the following flags: {}".format(
        ", ".join(strict_flag_names))
    strictness_group.add_argument(
        '--strict', action='store_true', dest='special-opts:strict',
        help=strict_help)

    report_group = parser.add_argument_group(
        title='Report generation',
        description='Generate a report in the specified format.')
    for report_type in sorted(defaults.REPORTER_NAMES):
        report_group.add_argument('--%s-report' % report_type.replace('_', '-'),
                                  metavar='DIR',
                                  dest='special-opts:%s_report' % report_type)

    other_group = parser.add_argument_group(
        title='Miscellaneous')
    other_group.add_argument(
        '--quickstart-file', help=argparse.SUPPRESS)
    other_group.add_argument(
        '--junit-xml', help="Write junit.xml to the given file")
    other_group.add_argument(
        '--scripts-are-modules', action='store_true',
        help="Script x becomes module x instead of __main__")
    other_group.add_argument(
        '--find-occurrences', metavar='CLASS.MEMBER',
        dest='special-opts:find_occurrences',
        help="Print out all usages of a class member (experimental)")

    if server_options:
        # TODO: This flag is superfluous; remove after a short transition (2018-03-16)
        other_group.add_argument(
            '--experimental', action='store_true', dest='fine_grained_incremental',
            help="Enable fine-grained incremental mode")
        other_group.add_argument(
            '--use-fine-grained-cache', action='store_true',
            help="Use the cache in fine-grained incremental mode")

    # hidden options
    parser.add_argument(
        '--stats', action='store_true', dest='dump_type_stats', help=argparse.SUPPRESS)
    parser.add_argument(
        '--inferstats', action='store_true', dest='dump_inference_stats',
        help=argparse.SUPPRESS)
    # --debug-cache will disable any cache-related compressions/optimizations,
    # which will make the cache writing process output pretty-printed JSON (which
    # is easier to debug).
    parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS)
    # --dump-deps will dump all fine-grained dependencies to stdout
    parser.add_argument('--dump-deps', action='store_true', help=argparse.SUPPRESS)
    # --dump-graph will dump the contents of the graph of SCCs and exit.
    parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS)
    # --semantic-analysis-only does exactly that.
    parser.add_argument('--semantic-analysis-only', action='store_true', help=argparse.SUPPRESS)
    # --local-partial-types disallows partial types spanning module top level and a function
    # (implicitly defined in fine-grained incremental mode)
    parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
    # --logical-deps adds some more dependencies that are not semantically needed, but
    # may be helpful to determine relative importance of classes and functions for overall
    # type precision in a code base. It also _removes_ some deps, so this flag should be never
    # used except for generating code stats. This also automatically enables --cache-fine-grained.
    # NOTE: This is an experimental option that may be modified or removed at any time.
    parser.add_argument('--logical-deps', action='store_true', help=argparse.SUPPRESS)
    # --bazel changes some behaviors for use with Bazel (https://bazel.build).
    parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
    # --package-root adds a directory below which directories are considered
    # packages even without __init__.py.  May be repeated.
    parser.add_argument('--package-root', metavar='ROOT', action='append', default=[],
                        help=argparse.SUPPRESS)
    # --cache-map FILE ... gives a mapping from source files to cache files.
    # Each triple of arguments is a source file, a cache meta file, and a cache data file.
    # Modules not mentioned in the file will go through cache_dir.
    # Must be followed by another flag or by '--' (and then only file args may follow).
    parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map',
                        help=argparse.SUPPRESS)

    # options specifying code to check
    code_group = parser.add_argument_group(
        title="Running code",
        description="Specify the code you want to type check. For more details, see "
                    "mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy")
    code_group.add_argument(
        '-m', '--module', action='append', metavar='MODULE',
        default=[],
        dest='special-opts:modules',
        help="Type-check module; can repeat for more modules")
    code_group.add_argument(
        '-p', '--package', action='append', metavar='PACKAGE',
        default=[],
        dest='special-opts:packages',
        help="Type-check package recursively; can be repeated")
    code_group.add_argument(
        '-c', '--command', action='append', metavar='PROGRAM_TEXT',
        dest='special-opts:command',
        help="Type-check program passed in as string")
    code_group.add_argument(
        metavar='files', nargs='*', dest='special-opts:files',
        help="Type-check given files or directories")

    # Parse arguments once into a dummy namespace so we can get the
    # filename for the config file and know if the user requested all strict options.
    dummy = argparse.Namespace()
    parser.parse_args(args, dummy)
    config_file = dummy.config_file
    # Don't explicitly test if "config_file is not None" for this check.
    # This lets `--config-file=` (an empty string) be used to disable all config files.
    if config_file and not os.path.exists(config_file):
        parser.error("Cannot find config file '%s'" % config_file)

    # Parse config file first, so command line can override.
    options = Options()
    parse_config_file(options, config_file, stdout, stderr)

    # Set strict flags before parsing (if strict mode enabled), so other command
    # line options can override.
    if getattr(dummy, 'special-opts:strict'):  # noqa
        for dest, value in strict_flag_assignments:
            setattr(options, dest, value)

    # Override cache_dir if provided in the environment
    environ_cache_dir = os.getenv('MYPY_CACHE_DIR', '')
    if environ_cache_dir.strip():
        options.cache_dir = environ_cache_dir

    # Parse command line for real, using a split namespace.
    special_opts = argparse.Namespace()
    parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:'))

    # The python_version is either the default, which can be overridden via a config file,
    # or stored in special_opts and is passed via the command line.
    options.python_version = special_opts.python_version or options.python_version
    try:
        infer_python_executable(options, special_opts)
    except PythonExecutableInferenceError as e:
        parser.error(str(e))

    if special_opts.no_executable:
        options.python_executable = None

    # Paths listed in the config file will be ignored if any paths are passed on
    # the command line.
    if options.files and not special_opts.files:
        special_opts.files = options.files

    # Check for invalid argument combinations.
    if require_targets:
        code_methods = sum(bool(c) for c in [special_opts.modules + special_opts.packages,
                                             special_opts.command,
                                             special_opts.files])
        if code_methods == 0:
            parser.error("Missing target module, package, files, or command.")
        elif code_methods > 1:
            parser.error("May only specify one of: module/package, files, or command.")

    # Check for overlapping `--always-true` and `--always-false` flags.
    overlap = set(options.always_true) & set(options.always_false)
    if overlap:
        parser.error("You can't make a variable always true and always false (%s)" %
                     ', '.join(sorted(overlap)))

    # Set build flags.
    if options.strict_optional_whitelist is not None:
        # TODO: Deprecate, then kill this flag
        options.strict_optional = True
    if special_opts.find_occurrences:
        state.find_occurrences = special_opts.find_occurrences.split('.')
        assert state.find_occurrences is not None
        if len(state.find_occurrences) < 2:
            parser.error("Can only find occurrences of class members.")
        if len(state.find_occurrences) != 2:
            parser.error("Can only find occurrences of non-nested class members.")

    # Set reports.
    for flag, val in vars(special_opts).items():
        if flag.endswith('_report') and val is not None:
            report_type = flag[:-7].replace('_', '-')
            report_dir = val
            options.report_dirs[report_type] = report_dir

    # Process --package-root.
    if options.package_root:
        process_package_roots(fscache, parser, options)

    # Process --cache-map.
    if special_opts.cache_map:
        if options.sqlite_cache:
            parser.error("--cache-map is incompatible with --sqlite-cache")

        process_cache_map(parser, special_opts, options)

    # Let logical_deps imply cache_fine_grained (otherwise the former is useless).
    if options.logical_deps:
        options.cache_fine_grained = True

    # Set target.
    if special_opts.modules + special_opts.packages:
        options.build_type = BuildType.MODULE
        search_paths = SearchPaths((os.getcwd(),), tuple(mypy_path() + options.mypy_path), (), ())
        targets = []
        # TODO: use the same cache that the BuildManager will
        cache = FindModuleCache(search_paths, fscache, options, special_opts.packages)
        for p in special_opts.packages:
            if os.sep in p or os.altsep and os.altsep in p:
                fail("Package name '{}' cannot have a slash in it.".format(p),
                     stderr)
            p_targets = cache.find_modules_recursive(p)
            if not p_targets:
                fail("Can't find package '{}'".format(p), stderr)
            targets.extend(p_targets)
        for m in special_opts.modules:
            targets.append(BuildSource(None, m, None))
        return targets, options
    elif special_opts.command:
        options.build_type = BuildType.PROGRAM_TEXT
        targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
        return targets, options
    else:
        try:
            targets = create_source_list(special_opts.files, options, fscache)
        # Variable named e2 instead of e to work around mypyc bug #620
        # which causes issues when using the same variable to catch
        # exceptions of different types.
        except InvalidSourceList as e2:
            fail(str(e2), stderr)
        return targets, options


def process_package_roots(fscache: Optional[FileSystemCache],
                          parser: argparse.ArgumentParser,
                          options: Options) -> None:
    """Validate and normalize package_root."""
    if fscache is None:
        parser.error("--package-root does not work here (no fscache)")
    assert fscache is not None  # Since mypy doesn't know parser.error() raises.
    # Do some stuff with drive letters to make Windows happy (esp. tests).
    current_drive, _ = os.path.splitdrive(os.getcwd())
    dot = os.curdir
    dotslash = os.curdir + os.sep
    dotdotslash = os.pardir + os.sep
    trivial_paths = {dot, dotslash}
    package_root = []
    for root in options.package_root:
        if os.path.isabs(root):
            parser.error("Package root cannot be absolute: %r" % root)
        drive, root = os.path.splitdrive(root)
        if drive and drive != current_drive:
            parser.error("Package root must be on current drive: %r" % (drive + root))
        # Empty package root is always okay.
        if root:
            root = os.path.relpath(root)  # Normalize the heck out of it.
            if root.startswith(dotdotslash):
                parser.error("Package root cannot be above current directory: %r" % root)
            if root in trivial_paths:
                root = ''
            elif not root.endswith(os.sep):
                root = root + os.sep
        package_root.append(root)
    options.package_root = package_root
    # Pass the package root on the the filesystem cache.
    fscache.set_package_root(package_root)


def process_cache_map(parser: argparse.ArgumentParser,
                      special_opts: argparse.Namespace,
                      options: Options) -> None:
    """Validate cache_map and copy into options.cache_map."""
    n = len(special_opts.cache_map)
    if n % 3 != 0:
        parser.error("--cache-map requires one or more triples (see source)")
    for i in range(0, n, 3):
        source, meta_file, data_file = special_opts.cache_map[i:i + 3]
        if source in options.cache_map:
            parser.error("Duplicate --cache-map source %s)" % source)
        if not source.endswith('.py') and not source.endswith('.pyi'):
            parser.error("Invalid --cache-map source %s (triple[0] must be *.py[i])" % source)
        if not meta_file.endswith('.meta.json'):
            parser.error("Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)" %
                         meta_file)
        if not data_file.endswith('.data.json'):
            parser.error("Invalid --cache-map data_file %s (triple[2] must be *.data.json)" %
                         data_file)
        options.cache_map[source] = (meta_file, data_file)


def fail(msg: str, stderr: TextIO) -> None:
    stderr.write('%s\n' % msg)
    sys.exit(2)