build_ext.py

import os
import sys
import itertools
from importlib.machinery import EXTENSION_SUFFIXES
from importlib.util import cache_from_source as _compiled_file_name
from typing import Dict, Iterator, List, Tuple
from pathlib import Path

from distutils.command.build_ext import build_ext as _du_build_ext
from distutils.ccompiler import new_compiler
from distutils.sysconfig import customize_compiler, get_config_var
from distutils import log

from setuptools.errors import BaseError
from setuptools.extension import Extension, Library, PreprocessedExtension

try:
    # Attempt to use Cython for building extensions, if available
    from Cython.Distutils.build_ext import build_ext as _build_ext

    # Additionally, assert that the compiler module will load
    # also. Ref #1229.
    __import__('Cython.Compiler.Main')
except ImportError:
    _build_ext = _du_build_ext

# make sure _config_vars is initialized
get_config_var("LDSHARED")
from distutils.sysconfig import _config_vars as _CONFIG_VARS  # noqa


def _customize_compiler_for_shlib(compiler):
    if sys.platform == "darwin":
        # building .dylib requires additional compiler flags on OSX; here we
        # temporarily substitute the pyconfig.h variables so that distutils'
        # 'customize_compiler' uses them before we build the shared libraries.
        tmp = _CONFIG_VARS.copy()
        try:
            # XXX Help!  I don't have any idea whether these are right...
            _CONFIG_VARS[
                'LDSHARED'
            ] = "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup"
            _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
            _CONFIG_VARS['SO'] = ".dylib"
            customize_compiler(compiler)
        finally:
            _CONFIG_VARS.clear()
            _CONFIG_VARS.update(tmp)
    else:
        customize_compiler(compiler)


have_rtld = False
use_stubs = False
libtype = 'shared'

if sys.platform == "darwin":
    use_stubs = True
elif os.name != 'nt':
    try:
        import dl

        use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')
    except ImportError:
        pass


def if_dl(s):
    return s if have_rtld else ''


def get_abi3_suffix():
    """Return the file extension for an abi3-compliant Extension()"""
    for suffix in EXTENSION_SUFFIXES:
        if '.abi3' in suffix:  # Unix
            return suffix
        elif suffix == '.pyd':  # Windows
            return suffix


class build_ext(_build_ext):
    editable_mode: bool = False
    inplace: bool = False

    def run(self):
        """Build extensions in build directory, then copy if --inplace"""
        old_inplace, self.inplace = self.inplace, 0
        _build_ext.run(self)
        self.inplace = old_inplace
        if old_inplace:
            self.copy_extensions_to_source()

    def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]:
        fullname = self.get_ext_fullname(ext.name)
        filename = self.get_ext_filename(fullname)
        modpath = fullname.split('.')
        package = '.'.join(modpath[:-1])
        package_dir = build_py.get_package_dir(package)
        inplace_file = os.path.join(package_dir, os.path.basename(filename))
        regular_file = os.path.join(self.build_lib, filename)
        return (inplace_file, regular_file)

    def copy_extensions_to_source(self):
        build_py = self.get_finalized_command('build_py')
        for ext in self.extensions:
            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)

            # Always copy, even if source is older than destination, to ensure
            # that the right extensions for the current Python/platform are
            # used.
            if os.path.exists(regular_file) or not ext.optional:
                self.copy_file(regular_file, inplace_file, level=self.verbose)

            if ext._needs_stub:
                inplace_stub = self._get_equivalent_stub(ext, inplace_file)
                self._write_stub_file(inplace_stub, ext, compile=True)
                # Always compile stub and remove the original (leave the cache behind)
                # (this behaviour was observed in previous iterations of the code)

    def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str:
        dir_ = os.path.dirname(output_file)
        _, _, name = ext.name.rpartition(".")
        return f"{os.path.join(dir_, name)}.py"

    def _get_output_mapping(self) -> Iterator[Tuple[str, str]]:
        if not self.inplace:
            return

        build_py = self.get_finalized_command('build_py')
        opt = self.get_finalized_command('install_lib').optimize or ""

        for ext in self.extensions:
            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)
            yield (regular_file, inplace_file)

            if ext._needs_stub:
                # This version of `build_ext` always builds artifacts in another dir,
                # when "inplace=True" is given it just copies them back.
                # This is done in the `copy_extensions_to_source` function, which
                # always compile stub files via `_compile_and_remove_stub`.
                # At the end of the process, a `.pyc` stub file is created without the
                # corresponding `.py`.

                inplace_stub = self._get_equivalent_stub(ext, inplace_file)
                regular_stub = self._get_equivalent_stub(ext, regular_file)
                inplace_cache = _compiled_file_name(inplace_stub, optimization=opt)
                output_cache = _compiled_file_name(regular_stub, optimization=opt)
                yield (output_cache, inplace_cache)

    def get_ext_filename(self, fullname):
        so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX')
        if so_ext:
            filename = os.path.join(*fullname.split('.')) + so_ext
        else:
            filename = _build_ext.get_ext_filename(self, fullname)
            so_ext = get_config_var('EXT_SUFFIX')

        if fullname in self.ext_map:
            ext = self.ext_map[fullname]
            use_abi3 = getattr(ext, 'py_limited_api') and get_abi3_suffix()
            if use_abi3:
                filename = filename[: -len(so_ext)]
                so_ext = get_abi3_suffix()
                filename = filename + so_ext
            if isinstance(ext, Library):
                fn, ext = os.path.splitext(filename)
                return self.shlib_compiler.library_filename(fn, libtype)
            elif use_stubs and ext._links_to_dynamic:
                d, fn = os.path.split(filename)
                return os.path.join(d, 'dl-' + fn)
        return filename

    def initialize_options(self):
        _build_ext.initialize_options(self)
        self.shlib_compiler = None
        self.shlibs = []
        self.ext_map = {}
        self.editable_mode = False

    def finalize_options(self):
        _build_ext.finalize_options(self)
        self.extensions = self.extensions or []
        self.check_extensions_list(self.extensions)
        self.shlibs = [ext for ext in self.extensions if isinstance(ext, Library)]
        if self.shlibs:
            self.setup_shlib_compiler()

        for ext in self.extensions:
            ext.__dict__.update(self._update_ext_info(ext))
            fullname = ext._full_name
            self.ext_map[fullname] = ext

            # distutils 3.1 will also ask for module names
            # XXX what to do with conflicts?
            self.ext_map[fullname.split('.')[-1]] = ext

        if self.editable_mode:
            self.inplace = True

    def _update_ext_info(self, ext: Extension) -> dict:
        """Setuptools needs to add extra attributes to build extension objects"""
        full_name = self.get_ext_fullname(ext.name)
        file_name = self.get_ext_filename(full_name)
        ltd = self.shlibs and self.links_to_dynamic(ext) or False
        ns = ltd and use_stubs and not isinstance(ext, Library)
        libdir = os.path.dirname(os.path.join(self.build_lib, file_name))

        updates = {
            "_file_name": file_name,
            "_full_name": full_name,
            "_links_to_dynamic": ltd,
            "_needs_stub": ns,
        }

        if ltd and libdir not in ext.library_dirs:
            updates["library_dirs"] = [*ext.library_dirs, libdir]
        if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
            updates["runtime_library_dirs"] = [*ext.runtime_library_dirs, os.curdir]

        return updates

    def setup_shlib_compiler(self):
        compiler = self.shlib_compiler = new_compiler(
            compiler=self.compiler, dry_run=self.dry_run, force=self.force
        )
        _customize_compiler_for_shlib(compiler)

        if self.include_dirs is not None:
            compiler.set_include_dirs(self.include_dirs)
        if self.define is not None:
            # 'define' option is a list of (name,value) tuples
            for name, value in self.define:
                compiler.define_macro(name, value)
        if self.undef is not None:
            for macro in self.undef:
                compiler.undefine_macro(macro)
        if self.libraries is not None:
            compiler.set_libraries(self.libraries)
        if self.library_dirs is not None:
            compiler.set_library_dirs(self.library_dirs)
        if self.rpath is not None:
            compiler.set_runtime_library_dirs(self.rpath)
        if self.link_objects is not None:
            compiler.set_link_objects(self.link_objects)

        # hack so distutils' build_extension() builds a library instead
        compiler.link_shared_object = link_shared_object.__get__(compiler)

    def get_export_symbols(self, ext):
        if isinstance(ext, Library):
            return ext.export_symbols
        return _build_ext.get_export_symbols(self, ext)

    def _preprocess(self, ext):
        if isinstance(ext, PreprocessedExtension):
            target = ext.preprocess(self)  # may be missing build info
            updates = self._update_ext_info(target)
            target.__dict__.update(updates)
            ext.__dict__.update(updates)  # ... for the sake of consistency ...
            return target

        return ext

    def build_extensions(self):
        self.extensions[:] = [self._preprocess(ext) for ext in self.extensions]
        # ^-- We may have to pre-process all the extensions first, before proceeding
        #     to the compilation step, since they may generate files that depend
        #     at at other during compile-time (e.g. headers).
        super().build_extensions()

    def build_extension(self, ext):
        ext._convert_pyx_sources_to_lang()
        _compiler = self.compiler
        try:
            if isinstance(ext, Library):
                self.compiler = self.shlib_compiler
            _build_ext.build_extension(self, ext)
            if ext._needs_stub:
                build_lib = self.get_finalized_command('build_py').build_lib
                self.write_stub(build_lib, ext)
        finally:
            self.compiler = _compiler

    def links_to_dynamic(self, ext):
        """Return true if 'ext' links to a dynamic lib in the same package"""
        if not (ext.libraries and self.shlibs):
            return False  # avoid calculations if not necessary

        # XXX this should check to ensure the lib is actually being built
        # XXX as dynamic, and not just using a locally-found version or a
        # XXX static-compiled version
        libnames = {self.get_ext_fullname(lib.name) for lib in self.shlibs}
        pkg = '.'.join(self.get_ext_fullname(ext.name).split('.')[:-1] + [''])
        return any(pkg + libname in libnames for libname in ext.libraries)

    def get_source_files(self) -> List[str]:
        return [*_build_ext.get_source_files(self), *self._get_internal_depends()]

    def _get_internal_depends(self) -> Iterator[str]:
        """Yield ``ext.depends`` that are contained by the project directory"""
        project_root = Path(self.distribution.src_root or os.curdir).resolve()
        depends = (dep for ext in self.extensions for dep in ext.depends)

        def skip(orig_path: str, reason: str) -> None:
            log.info(
                "dependency %s won't be automatically "
                "included in the manifest: the path %s",
                orig_path,
                reason,
            )

        for dep in depends:
            path = Path(dep)

            if path.is_absolute():
                skip(dep, "must be relative")
                continue

            if ".." in path.parts:
                skip(dep, "can't have `..` segments")
                continue

            try:
                resolved = (project_root / path).resolve(strict=True)
            except OSError:
                skip(dep, "doesn't exist")
                continue

            try:
                resolved.relative_to(project_root)
            except ValueError:
                skip(dep, "must be inside the project root")
                continue

            yield path.as_posix()

    def get_outputs(self) -> List[str]:
        if self.inplace:
            return list(self.get_output_mapping().keys())
        return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs())

    def get_output_mapping(self) -> Dict[str, str]:
        """See :class:`setuptools.commands.build.SubCommand`"""
        mapping = self._get_output_mapping()
        return dict(sorted(mapping, key=lambda x: x[0]))

    def __get_stubs_outputs(self):
        # assemble the base name for each extension that needs a stub
        ns_ext_bases = (
            os.path.join(self.build_lib, *ext._full_name.split('.'))
            for ext in self.extensions
            if ext._needs_stub
        )
        # pair each base with the extension
        pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())
        return list(base + fnext for base, fnext in pairs)

    def __get_output_extensions(self):
        yield '.py'
        yield '.pyc'
        if self.get_finalized_command('build_py').optimize:
            yield '.pyo'

    def write_stub(self, output_dir, ext, compile=False):
        stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py'
        self._write_stub_file(stub_file, ext, compile)

    def _write_stub_file(self, stub_file: str, ext: Extension, compile=False):
        log.info("writing stub loader for %s to %s", ext._full_name, stub_file)
        if compile and os.path.exists(stub_file):
            raise BaseError(stub_file + " already exists! Please delete.")
        if not self.dry_run:
            f = open(stub_file, 'w')
            f.write(
                '\n'.join(
                    [
                        "def __bootstrap__():",
                        "   global __bootstrap__, __file__, __loader__",
                        "   import sys, os, pkg_resources, importlib.util"
                        + if_dl(", dl"),
                        "   __file__ = pkg_resources.resource_filename"
                        "(__name__,%r)" % os.path.basename(ext._file_name),
                        "   del __bootstrap__",
                        "   if '__loader__' in globals():",
                        "       del __loader__",
                        if_dl("   old_flags = sys.getdlopenflags()"),
                        "   old_dir = os.getcwd()",
                        "   try:",
                        "     os.chdir(os.path.dirname(__file__))",
                        if_dl("     sys.setdlopenflags(dl.RTLD_NOW)"),
                        "     spec = importlib.util.spec_from_file_location(",
                        "                __name__, __file__)",
                        "     mod = importlib.util.module_from_spec(spec)",
                        "     spec.loader.exec_module(mod)",
                        "   finally:",
                        if_dl("     sys.setdlopenflags(old_flags)"),
                        "     os.chdir(old_dir)",
                        "__bootstrap__()",
                        "",  # terminal \n
                    ]
                )
            )
            f.close()
        if compile:
            self._compile_and_remove_stub(stub_file)

    def _compile_and_remove_stub(self, stub_file: str):
        from distutils.util import byte_compile

        byte_compile([stub_file], optimize=0, force=True, dry_run=self.dry_run)
        optimize = self.get_finalized_command('install_lib').optimize
        if optimize > 0:
            byte_compile(
                [stub_file], optimize=optimize, force=True, dry_run=self.dry_run
            )
        if os.path.exists(stub_file) and not self.dry_run:
            os.unlink(stub_file)


if use_stubs or os.name == 'nt':
    # Build shared libraries
    #
    def link_shared_object(
        self,
        objects,
        output_libname,
        output_dir=None,
        libraries=None,
        library_dirs=None,
        runtime_library_dirs=None,
        export_symbols=None,
        debug=0,
        extra_preargs=None,
        extra_postargs=None,
        build_temp=None,
        target_lang=None,
    ):
        self.link(
            self.SHARED_LIBRARY,
            objects,
            output_libname,
            output_dir,
            libraries,
            library_dirs,
            runtime_library_dirs,
            export_symbols,
            debug,
            extra_preargs,
            extra_postargs,
            build_temp,
            target_lang,
        )

else:
    # Build static libraries everywhere else
    libtype = 'static'

    def link_shared_object(
        self,
        objects,
        output_libname,
        output_dir=None,
        libraries=None,
        library_dirs=None,
        runtime_library_dirs=None,
        export_symbols=None,
        debug=0,
        extra_preargs=None,
        extra_postargs=None,
        build_temp=None,
        target_lang=None,
    ):
        # XXX we need to either disallow these attrs on Library instances,
        # or warn/abort here if set, or something...
        # libraries=None, library_dirs=None, runtime_library_dirs=None,
        # export_symbols=None, extra_preargs=None, extra_postargs=None,
        # build_temp=None

        assert output_dir is None  # distutils build_ext doesn't pass this
        output_dir, filename = os.path.split(output_libname)
        basename, ext = os.path.splitext(filename)
        if self.library_filename("x").startswith('lib'):
            # strip 'lib' prefix; this is kludgy if some platform uses
            # a different prefix
            basename = basename[3:]

        self.create_static_lib(objects, basename, output_dir, debug, target_lang)