stubgenc.py

#!/usr/bin/env python3
"""Stub generator for C modules.

The public interface is via the mypy.stubgen module.
"""

import importlib
import inspect
import os.path
import re
from typing import List, Dict, Tuple, Optional, Mapping, Any, Set
from types import ModuleType
from typing_extensions import Final

from mypy.moduleinspect import is_c_module
from mypy.stubdoc import (
    infer_sig_from_docstring, infer_prop_type_from_docstring, ArgSig,
    infer_arg_sig_from_anon_docstring, infer_ret_type_sig_from_anon_docstring,
    infer_ret_type_sig_from_docstring, FunctionSig
)

# Members of the typing module to consider for importing by default.
_DEFAULT_TYPING_IMPORTS: Final = (
    'Any',
    'Callable',
    'ClassVar',
    'Dict',
    'Iterable',
    'Iterator',
    'List',
    'Optional',
    'Tuple',
    'Union',
)


def generate_stub_for_c_module(module_name: str,
                               target: str,
                               sigs: Optional[Dict[str, str]] = None,
                               class_sigs: Optional[Dict[str, str]] = None) -> None:
    """Generate stub for C module.

    This combines simple runtime introspection (looking for docstrings and attributes
    with simple builtin types) and signatures inferred from .rst documentation (if given).

    If directory for target doesn't exist it will be created. Existing stub
    will be overwritten.
    """
    module = importlib.import_module(module_name)
    assert is_c_module(module), f'{module_name} is not a C module'
    subdir = os.path.dirname(target)
    if subdir and not os.path.isdir(subdir):
        os.makedirs(subdir)
    imports: List[str] = []
    functions: List[str] = []
    done = set()
    items = sorted(module.__dict__.items(), key=lambda x: x[0])
    for name, obj in items:
        if is_c_function(obj):
            generate_c_function_stub(module, name, obj, functions, imports=imports, sigs=sigs)
            done.add(name)
    types: List[str] = []
    for name, obj in items:
        if name.startswith('__') and name.endswith('__'):
            continue
        if is_c_type(obj):
            generate_c_type_stub(module, name, obj, types, imports=imports, sigs=sigs,
                                 class_sigs=class_sigs)
            done.add(name)
    variables = []
    for name, obj in items:
        if name.startswith('__') and name.endswith('__'):
            continue
        if name not in done and not inspect.ismodule(obj):
            type_str = strip_or_import(get_type_fullname(type(obj)), module, imports)
            variables.append(f'{name}: {type_str}')
    output = []
    for line in sorted(set(imports)):
        output.append(line)
    for line in variables:
        output.append(line)
    for line in types:
        if line.startswith('class') and output and output[-1]:
            output.append('')
        output.append(line)
    if output and functions:
        output.append('')
    for line in functions:
        output.append(line)
    output = add_typing_import(output)
    with open(target, 'w') as file:
        for line in output:
            file.write(f'{line}\n')


def add_typing_import(output: List[str]) -> List[str]:
    """Add typing imports for collections/types that occur in the generated stub."""
    names = []
    for name in _DEFAULT_TYPING_IMPORTS:
        if any(re.search(r'\b%s\b' % name, line) for line in output):
            names.append(name)
    if names:
        return [f"from typing import {', '.join(names)}", ''] + output
    else:
        return output[:]


def is_c_function(obj: object) -> bool:
    return inspect.isbuiltin(obj) or type(obj) is type(ord)


def is_c_method(obj: object) -> bool:
    return inspect.ismethoddescriptor(obj) or type(obj) in (type(str.index),
                                                            type(str.__add__),
                                                            type(str.__new__))


def is_c_classmethod(obj: object) -> bool:
    return inspect.isbuiltin(obj) or type(obj).__name__ in ('classmethod',
                                                            'classmethod_descriptor')


def is_c_property(obj: object) -> bool:
    return inspect.isdatadescriptor(obj) or hasattr(obj, 'fget')


def is_c_property_readonly(prop: Any) -> bool:
    return hasattr(prop, 'fset') and prop.fset is None


def is_c_type(obj: object) -> bool:
    return inspect.isclass(obj) or type(obj) is type(int)


def is_pybind11_overloaded_function_docstring(docstr: str, name: str) -> bool:
    return docstr.startswith(f"{name}(*args, **kwargs)\n" +
                             "Overloaded function.\n\n")


def generate_c_function_stub(module: ModuleType,
                             name: str,
                             obj: object,
                             output: List[str],
                             imports: List[str],
                             self_var: Optional[str] = None,
                             sigs: Optional[Dict[str, str]] = None,
                             class_name: Optional[str] = None,
                             class_sigs: Optional[Dict[str, str]] = None) -> None:
    """Generate stub for a single function or method.

    The result (always a single line) will be appended to 'output'.
    If necessary, any required names will be added to 'imports'.
    The 'class_name' is used to find signature of __init__ or __new__ in
    'class_sigs'.
    """
    if sigs is None:
        sigs = {}
    if class_sigs is None:
        class_sigs = {}

    ret_type = 'None' if name == '__init__' and class_name else 'Any'

    if (
        name in ("__new__", "__init__")
        and name not in sigs
        and class_name
        and class_name in class_sigs
    ):
        inferred: Optional[List[FunctionSig]] = [
            FunctionSig(
                name=name,
                args=infer_arg_sig_from_anon_docstring(class_sigs[class_name]),
                ret_type=ret_type,
            )
        ]
    else:
        docstr = getattr(obj, '__doc__', None)
        inferred = infer_sig_from_docstring(docstr, name)
        if inferred:
            assert docstr is not None
            if is_pybind11_overloaded_function_docstring(docstr, name):
                # Remove pybind11 umbrella (*args, **kwargs) for overloaded functions
                del inferred[-1]
        if not inferred:
            if class_name and name not in sigs:
                inferred = [FunctionSig(name, args=infer_method_sig(name, self_var),
                                        ret_type=ret_type)]
            else:
                inferred = [FunctionSig(name=name,
                                        args=infer_arg_sig_from_anon_docstring(
                                            sigs.get(name, '(*args, **kwargs)')),
                                        ret_type=ret_type)]
        elif class_name and self_var:
            args = inferred[0].args
            if not args or args[0].name != self_var:
                args.insert(0, ArgSig(name=self_var))

    is_overloaded = len(inferred) > 1 if inferred else False
    if is_overloaded:
        imports.append('from typing import overload')
    if inferred:
        for signature in inferred:
            sig = []
            for arg in signature.args:
                if arg.name == self_var:
                    arg_def = self_var
                else:
                    arg_def = arg.name
                    if arg_def == 'None':
                        arg_def = '_none'  # None is not a valid argument name

                    if arg.type:
                        arg_def += ": " + strip_or_import(arg.type, module, imports)

                    if arg.default:
                        arg_def += " = ..."

                sig.append(arg_def)

            if is_overloaded:
                output.append('@overload')
            output.append('def {function}({args}) -> {ret}: ...'.format(
                function=name,
                args=", ".join(sig),
                ret=strip_or_import(signature.ret_type, module, imports)
            ))


def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str:
    """Strips unnecessary module names from typ.

    If typ represents a type that is inside module or is a type coming from builtins, remove
    module declaration from it. Return stripped name of the type.

    Arguments:
        typ: name of the type
        module: in which this type is used
        imports: list of import statements (may be modified during the call)
    """
    stripped_type = typ
    if any(c in typ for c in '[,'):
        for subtyp in re.split(r'[\[,\]]', typ):
            strip_or_import(subtyp.strip(), module, imports)
        if module:
            stripped_type = re.sub(
                r'(^|[\[, ]+)' + re.escape(module.__name__ + '.'),
                r'\1',
                typ,
            )
    elif module and typ.startswith(module.__name__ + '.'):
        stripped_type = typ[len(module.__name__) + 1:]
    elif '.' in typ:
        arg_module = typ[:typ.rindex('.')]
        if arg_module == 'builtins':
            stripped_type = typ[len('builtins') + 1:]
        else:
            imports.append(f'import {arg_module}')
    if stripped_type == 'NoneType':
        stripped_type = 'None'
    return stripped_type


def is_static_property(obj: object) -> bool:
    return type(obj).__name__ == 'pybind11_static_property'


def generate_c_property_stub(name: str, obj: object,
                             static_properties: List[str],
                             rw_properties: List[str],
                             ro_properties: List[str], readonly: bool,
                             module: Optional[ModuleType] = None,
                             imports: Optional[List[str]] = None) -> None:
    """Generate property stub using introspection of 'obj'.

    Try to infer type from docstring, append resulting lines to 'output'.
    """

    def infer_prop_type(docstr: Optional[str]) -> Optional[str]:
        """Infer property type from docstring or docstring signature."""
        if docstr is not None:
            inferred = infer_ret_type_sig_from_anon_docstring(docstr)
            if not inferred:
                inferred = infer_ret_type_sig_from_docstring(docstr, name)
            if not inferred:
                inferred = infer_prop_type_from_docstring(docstr)
            return inferred
        else:
            return None

    # Ignore special properties/attributes.
    if is_skipped_attribute(name):
        return

    inferred = infer_prop_type(getattr(obj, '__doc__', None))
    if not inferred:
        fget = getattr(obj, 'fget', None)
        inferred = infer_prop_type(getattr(fget, '__doc__', None))
    if not inferred:
        inferred = 'Any'

    if module is not None and imports is not None:
        inferred = strip_or_import(inferred, module, imports)

    if is_static_property(obj):
        trailing_comment = "  # read-only" if readonly else ""
        static_properties.append(
            f'{name}: ClassVar[{inferred}] = ...{trailing_comment}'
        )
    else:  # regular property
        if readonly:
            ro_properties.append('@property')
            ro_properties.append(f'def {name}(self) -> {inferred}: ...')
        else:
            rw_properties.append(f'{name}: {inferred}')


def generate_c_type_stub(module: ModuleType,
                         class_name: str,
                         obj: type,
                         output: List[str],
                         imports: List[str],
                         sigs: Optional[Dict[str, str]] = None,
                         class_sigs: Optional[Dict[str, str]] = None) -> None:
    """Generate stub for a single class using runtime introspection.

    The result lines will be appended to 'output'. If necessary, any
    required names will be added to 'imports'.
    """
    # typeshed gives obj.__dict__ the not quite correct type Dict[str, Any]
    # (it could be a mappingproxy!), which makes mypyc mad, so obfuscate it.
    obj_dict: Mapping[str, Any] = getattr(obj, "__dict__")  # noqa
    items = sorted(obj_dict.items(), key=lambda x: method_name_sort_key(x[0]))
    methods: List[str] = []
    types: List[str] = []
    static_properties: List[str] = []
    rw_properties: List[str] = []
    ro_properties: List[str] = []
    done: Set[str] = set()
    for attr, value in items:
        if is_c_method(value) or is_c_classmethod(value):
            done.add(attr)
            if not is_skipped_attribute(attr):
                if attr == '__new__':
                    # TODO: We should support __new__.
                    if '__init__' in obj_dict:
                        # Avoid duplicate functions if both are present.
                        # But is there any case where .__new__() has a
                        # better signature than __init__() ?
                        continue
                    attr = '__init__'
                if is_c_classmethod(value):
                    methods.append('@classmethod')
                    self_var = 'cls'
                else:
                    self_var = 'self'
                generate_c_function_stub(module, attr, value, methods, imports=imports,
                                         self_var=self_var, sigs=sigs, class_name=class_name,
                                         class_sigs=class_sigs)
        elif is_c_property(value):
            done.add(attr)
            generate_c_property_stub(attr, value, static_properties, rw_properties, ro_properties,
                                     is_c_property_readonly(value),
                                     module=module, imports=imports)
        elif is_c_type(value):
            generate_c_type_stub(module, attr, value, types, imports=imports, sigs=sigs,
                                 class_sigs=class_sigs)
            done.add(attr)

    for attr, value in items:
        if is_skipped_attribute(attr):
            continue
        if attr not in done:
            static_properties.append('{}: ClassVar[{}] = ...'.format(
                attr, strip_or_import(get_type_fullname(type(value)), module, imports)))
    all_bases = type.mro(obj)
    if all_bases[-1] is object:
        # TODO: Is this always object?
        del all_bases[-1]
    # remove pybind11_object. All classes generated by pybind11 have pybind11_object in their MRO,
    # which only overrides a few functions in object type
    if all_bases and all_bases[-1].__name__ == 'pybind11_object':
        del all_bases[-1]
    # remove the class itself
    all_bases = all_bases[1:]
    # Remove base classes of other bases as redundant.
    bases: List[type] = []
    for base in all_bases:
        if not any(issubclass(b, base) for b in bases):
            bases.append(base)
    if bases:
        bases_str = '(%s)' % ', '.join(
            strip_or_import(
                get_type_fullname(base),
                module,
                imports
            ) for base in bases
        )
    else:
        bases_str = ''
    if types or static_properties or rw_properties or methods or ro_properties:
        output.append(f'class {class_name}{bases_str}:')
        for line in types:
            if output and output[-1] and \
                    not output[-1].startswith('class') and line.startswith('class'):
                output.append('')
            output.append('    ' + line)
        for line in static_properties:
            output.append(f'    {line}')
        for line in rw_properties:
            output.append(f'    {line}')
        for line in methods:
            output.append(f'    {line}')
        for line in ro_properties:
            output.append(f'    {line}')
    else:
        output.append(f'class {class_name}{bases_str}: ...')


def get_type_fullname(typ: type) -> str:
    return f"{typ.__module__}.{getattr(typ, '__qualname__', typ.__name__)}"


def method_name_sort_key(name: str) -> Tuple[int, str]:
    """Sort methods in classes in a typical order.

    I.e.: constructor, normal methods, special methods.
    """
    if name in ('__new__', '__init__'):
        return 0, name
    if name.startswith('__') and name.endswith('__'):
        return 2, name
    return 1, name


def is_pybind_skipped_attribute(attr: str) -> bool:
    return attr.startswith("__pybind11_module_local_")


def is_skipped_attribute(attr: str) -> bool:
    return (attr in ('__getattribute__',
                     '__str__',
                     '__repr__',
                     '__doc__',
                     '__dict__',
                     '__module__',
                     '__weakref__')  # For pickling
            or is_pybind_skipped_attribute(attr)
            )


def infer_method_sig(name: str, self_var: Optional[str] = None) -> List[ArgSig]:
    args: Optional[List[ArgSig]] = None
    if name.startswith('__') and name.endswith('__'):
        name = name[2:-2]
        if name in ('hash', 'iter', 'next', 'sizeof', 'copy', 'deepcopy', 'reduce', 'getinitargs',
                    'int', 'float', 'trunc', 'complex', 'bool', 'abs', 'bytes', 'dir', 'len',
                    'reversed', 'round', 'index', 'enter'):
            args = []
        elif name == 'getitem':
            args = [ArgSig(name='index')]
        elif name == 'setitem':
            args = [ArgSig(name='index'),
                    ArgSig(name='object')]
        elif name in ('delattr', 'getattr'):
            args = [ArgSig(name='name')]
        elif name == 'setattr':
            args = [ArgSig(name='name'),
                    ArgSig(name='value')]
        elif name == 'getstate':
            args = []
        elif name == 'setstate':
            args = [ArgSig(name='state')]
        elif name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge',
                      'add', 'radd', 'sub', 'rsub', 'mul', 'rmul',
                      'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv',
                      'divmod', 'rdivmod', 'pow', 'rpow',
                      'xor', 'rxor', 'or', 'ror', 'and', 'rand', 'lshift', 'rlshift',
                      'rshift', 'rrshift',
                      'contains', 'delitem',
                      'iadd', 'iand', 'ifloordiv', 'ilshift', 'imod', 'imul', 'ior',
                      'ipow', 'irshift', 'isub', 'itruediv', 'ixor'):
            args = [ArgSig(name='other')]
        elif name in ('neg', 'pos', 'invert'):
            args = []
        elif name == 'get':
            args = [ArgSig(name='instance'),
                    ArgSig(name='owner')]
        elif name == 'set':
            args = [ArgSig(name='instance'),
                    ArgSig(name='value')]
        elif name == 'reduce_ex':
            args = [ArgSig(name='protocol')]
        elif name == 'exit':
            args = [ArgSig(name='type'),
                    ArgSig(name='value'),
                    ArgSig(name='traceback')]
    if args is None:
        args = [ArgSig(name='*args'),
                ArgSig(name='**kwargs')]
    return [ArgSig(name=self_var or 'self')] + args