find_sources.py

"""Routines for finding the sources that mypy will check"""

import functools
import os

from typing import List, Sequence, Set, Tuple, Optional
from typing_extensions import Final

from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path
from mypy.fscache import FileSystemCache
from mypy.options import Options

PY_EXTENSIONS = tuple(PYTHON_EXTENSIONS)  # type: Final


class InvalidSourceList(Exception):
    """Exception indicating a problem in the list of sources given to mypy."""


def create_source_list(paths: Sequence[str], options: Options,
                       fscache: Optional[FileSystemCache] = None,
                       allow_empty_dir: bool = False) -> List[BuildSource]:
    """From a list of source files/directories, makes a list of BuildSources.

    Raises InvalidSourceList on errors.
    """
    fscache = fscache or FileSystemCache()
    finder = SourceFinder(fscache, options)

    sources = []
    for path in paths:
        path = os.path.normpath(path)
        if path.endswith(PY_EXTENSIONS):
            # Can raise InvalidSourceList if a directory doesn't have a valid module name.
            name, base_dir = finder.crawl_up(path)
            sources.append(BuildSource(path, name, None, base_dir))
        elif fscache.isdir(path):
            sub_sources = finder.find_sources_in_dir(path)
            if not sub_sources and not allow_empty_dir:
                raise InvalidSourceList(
                    "There are no .py[i] files in directory '{}'".format(path)
                )
            sources.extend(sub_sources)
        else:
            mod = os.path.basename(path) if options.scripts_are_modules else None
            sources.append(BuildSource(path, mod, None))
    return sources


def keyfunc(name: str) -> Tuple[bool, int, str]:
    """Determines sort order for directory listing.

    The desirable properties are:
    1) foo < foo.pyi < foo.py
    2) __init__.py[i] < foo
    """
    base, suffix = os.path.splitext(name)
    for i, ext in enumerate(PY_EXTENSIONS):
        if suffix == ext:
            return (base != "__init__", i, base)
    return (base != "__init__", -1, name)


def normalise_package_base(root: str) -> str:
    if not root:
        root = os.curdir
    root = os.path.abspath(root)
    if root.endswith(os.sep):
        root = root[:-1]
    return root


def get_explicit_package_bases(options: Options) -> Optional[List[str]]:
    """Returns explicit package bases to use if the option is enabled, or None if disabled.

    We currently use MYPYPATH and the current directory as the package bases. In the future,
    when --namespace-packages is the default could also use the values passed with the
    --package-root flag, see #9632.

    Values returned are normalised so we can use simple string comparisons in
    SourceFinder.is_explicit_package_base
    """
    if not options.explicit_package_bases:
        return None
    roots = mypy_path() + options.mypy_path + [os.getcwd()]
    return [normalise_package_base(root) for root in roots]


class SourceFinder:
    def __init__(self, fscache: FileSystemCache, options: Options) -> None:
        self.fscache = fscache
        self.explicit_package_bases = get_explicit_package_bases(options)
        self.namespace_packages = options.namespace_packages

    def is_explicit_package_base(self, path: str) -> bool:
        assert self.explicit_package_bases
        return normalise_package_base(path) in self.explicit_package_bases

    def find_sources_in_dir(self, path: str) -> List[BuildSource]:
        sources = []

        seen = set()  # type: Set[str]
        names = sorted(self.fscache.listdir(path), key=keyfunc)
        for name in names:
            # Skip certain names altogether
            if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
                continue
            subpath = os.path.join(path, name)

            if self.fscache.isdir(subpath):
                sub_sources = self.find_sources_in_dir(subpath)
                if sub_sources:
                    seen.add(name)
                    sources.extend(sub_sources)
            else:
                stem, suffix = os.path.splitext(name)
                if stem not in seen and suffix in PY_EXTENSIONS:
                    seen.add(stem)
                    module, base_dir = self.crawl_up(subpath)
                    sources.append(BuildSource(subpath, module, None, base_dir))

        return sources

    def crawl_up(self, path: str) -> Tuple[str, str]:
        """Given a .py[i] filename, return module and base directory.

        For example, given "xxx/yyy/foo/bar.py", we might return something like:
        ("foo.bar", "xxx/yyy")

        If namespace packages is off, we crawl upwards until we find a directory without
        an __init__.py

        If namespace packages is on, we crawl upwards until the nearest explicit base directory.
        Failing that, we return one past the highest directory containing an __init__.py

        We won't crawl past directories with invalid package names.
        The base directory returned is an absolute path.
        """
        path = os.path.abspath(path)
        parent, filename = os.path.split(path)

        module_name = strip_py(filename) or filename

        parent_module, base_dir = self.crawl_up_dir(parent)
        if module_name == "__init__":
            return parent_module, base_dir

        # Note that module_name might not actually be a valid identifier, but that's okay
        # Ignoring this possibility sidesteps some search path confusion
        module = module_join(parent_module, module_name)
        return module, base_dir

    def crawl_up_dir(self, dir: str) -> Tuple[str, str]:
        return self._crawl_up_helper(dir) or ("", dir)

    @functools.lru_cache()
    def _crawl_up_helper(self, dir: str) -> Optional[Tuple[str, str]]:
        """Given a directory, maybe returns module and base directory.

        We return a non-None value if we were able to find something clearly intended as a base
        directory (as adjudicated by being an explicit base directory or by containing a package
        with __init__.py).

        This distinction is necessary for namespace packages, so that we know when to treat
        ourselves as a subpackage.
        """
        # stop crawling if we're an explicit base directory
        if self.explicit_package_bases is not None and self.is_explicit_package_base(dir):
            return "", dir

        parent, name = os.path.split(dir)
        if name.endswith('-stubs'):
            name = name[:-6]  # PEP-561 stub-only directory

        # recurse if there's an __init__.py
        init_file = self.get_init_file(dir)
        if init_file is not None:
            if not name.isidentifier():
                # in most cases the directory name is invalid, we'll just stop crawling upwards
                # but if there's an __init__.py in the directory, something is messed up
                raise InvalidSourceList("{} is not a valid Python package name".format(name))
            # we're definitely a package, so we always return a non-None value
            mod_prefix, base_dir = self.crawl_up_dir(parent)
            return module_join(mod_prefix, name), base_dir

        # stop crawling if we're out of path components or our name is an invalid identifier
        if not name or not parent or not name.isidentifier():
            return None

        # stop crawling if namespace packages is off (since we don't have an __init__.py)
        if not self.namespace_packages:
            return None

        # at this point: namespace packages is on, we don't have an __init__.py and we're not an
        # explicit base directory
        result = self._crawl_up_helper(parent)
        if result is None:
            # we're not an explicit base directory and we don't have an __init__.py
            # and none of our parents are either, so return
            return None
        # one of our parents was an explicit base directory or had an __init__.py, so we're
        # definitely a subpackage! chain our name to the module.
        mod_prefix, base_dir = result
        return module_join(mod_prefix, name), base_dir

    def get_init_file(self, dir: str) -> Optional[str]:
        """Check whether a directory contains a file named __init__.py[i].

        If so, return the file's name (with dir prefixed).  If not, return None.

        This prefers .pyi over .py (because of the ordering of PY_EXTENSIONS).
        """
        for ext in PY_EXTENSIONS:
            f = os.path.join(dir, '__init__' + ext)
            if self.fscache.isfile(f):
                return f
            if ext == '.py' and self.fscache.init_under_package_root(f):
                return f
        return None


def module_join(parent: str, child: str) -> str:
    """Join module ids, accounting for a possibly empty parent."""
    if parent:
        return parent + '.' + child
    return child


def strip_py(arg: str) -> Optional[str]:
    """Strip a trailing .py or .pyi suffix.

    Return None if no such suffix is found.
    """
    for ext in PY_EXTENSIONS:
        if arg.endswith(ext):
            return arg[:-len(ext)]
    return None