expand_modules.py

import os
import sys
from typing import List, Pattern, Tuple, Union

from astroid import modutils

if sys.version_info >= (3, 8):
    from typing import Literal, TypedDict
else:
    from typing_extensions import Literal, TypedDict


class ModuleDescriptionDict(TypedDict):
    path: str
    name: str
    isarg: bool
    basepath: str
    basename: str


class ErrorDescriptionTest(TypedDict):
    key: Literal["fatal"]
    mod: str
    ex: Union[ImportError, SyntaxError]


def _modpath_from_file(filename, is_namespace, path=None):
    def _is_package_cb(path, parts):
        return modutils.check_modpath_has_init(path, parts) or is_namespace

    return modutils.modpath_from_file_with_callback(
        filename, path=path, is_package_cb=_is_package_cb
    )


def get_python_path(filepath: str) -> str:
    """TODO This get the python path with the (bad) assumption that there is always
    an __init__.py. This is not true since python 3.3 and is causing problem."""
    dirname = os.path.realpath(os.path.expanduser(filepath))
    if not os.path.isdir(dirname):
        dirname = os.path.dirname(dirname)
    while True:
        if not os.path.exists(os.path.join(dirname, "__init__.py")):
            return dirname
        old_dirname = dirname
        dirname = os.path.dirname(dirname)
        if old_dirname == dirname:
            return os.getcwd()


def _is_in_ignore_list_re(element: str, ignore_list_re: List[Pattern]) -> bool:
    """determines if the element is matched in a regex ignore-list"""
    for file_pattern in ignore_list_re:
        if file_pattern.match(element):
            return True
    return False


def expand_modules(
    files_or_modules: List[str],
    ignore_list: List[str],
    ignore_list_re: List[Pattern],
    ignore_list_paths_re: List[Pattern],
) -> Tuple[List[ModuleDescriptionDict], List[ErrorDescriptionTest]]:
    """take a list of files/modules/packages and return the list of tuple
    (file, module name) which have to be actually checked
    """
    result: List[ModuleDescriptionDict] = []
    errors: List[ErrorDescriptionTest] = []
    path = sys.path.copy()

    for something in files_or_modules:
        basename = os.path.basename(something)
        if (
            basename in ignore_list
            or _is_in_ignore_list_re(os.path.basename(something), ignore_list_re)
            or _is_in_ignore_list_re(something, ignore_list_paths_re)
        ):
            continue
        module_path = get_python_path(something)
        additional_search_path = [".", module_path] + path
        if os.path.exists(something):
            # this is a file or a directory
            try:
                modname = ".".join(
                    modutils.modpath_from_file(something, path=additional_search_path)
                )
            except ImportError:
                modname = os.path.splitext(basename)[0]
            if os.path.isdir(something):
                filepath = os.path.join(something, "__init__.py")
            else:
                filepath = something
        else:
            # suppose it's a module or package
            modname = something
            try:
                filepath = modutils.file_from_modpath(
                    modname.split("."), path=additional_search_path
                )
                if filepath is None:
                    continue
            except (ImportError, SyntaxError) as ex:
                # The SyntaxError is a Python bug and should be
                # removed once we move away from imp.find_module: https://bugs.python.org/issue10588
                errors.append({"key": "fatal", "mod": modname, "ex": ex})
                continue
        filepath = os.path.normpath(filepath)
        modparts = (modname or something).split(".")
        try:
            spec = modutils.file_info_from_modpath(
                modparts, path=additional_search_path
            )
        except ImportError:
            # Might not be acceptable, don't crash.
            is_namespace = False
            is_directory = os.path.isdir(something)
        else:
            is_namespace = modutils.is_namespace(spec)
            is_directory = modutils.is_directory(spec)
        if not is_namespace:
            result.append(
                {
                    "path": filepath,
                    "name": modname,
                    "isarg": True,
                    "basepath": filepath,
                    "basename": modname,
                }
            )
        has_init = (
            not (modname.endswith(".__init__") or modname == "__init__")
            and os.path.basename(filepath) == "__init__.py"
        )
        if has_init or is_namespace or is_directory:
            for subfilepath in modutils.get_module_files(
                os.path.dirname(filepath), ignore_list, list_all=is_namespace
            ):
                if filepath == subfilepath:
                    continue
                if _is_in_ignore_list_re(
                    os.path.basename(subfilepath), ignore_list_re
                ) or _is_in_ignore_list_re(subfilepath, ignore_list_paths_re):
                    continue

                modpath = _modpath_from_file(
                    subfilepath, is_namespace, path=additional_search_path
                )
                submodname = ".".join(modpath)
                result.append(
                    {
                        "path": subfilepath,
                        "name": submodname,
                        "isarg": False,
                        "basepath": filepath,
                        "basename": modname,
                    }
                )
    return result, errors