installer.py

import os
import shutil
import time

from conans.client import tools
from conans.client.file_copier import report_copied_files
from conans.client.generators import TXTGenerator, write_generators
from conans.client.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_EDITABLE, \
    BINARY_MISSING, BINARY_SKIP, BINARY_UPDATE, BINARY_UNKNOWN
from conans.client.importer import remove_imports, run_imports
from conans.client.packager import run_package_method, update_package_metadata
from conans.client.recorder.action_recorder import INSTALL_ERROR_BUILDING, INSTALL_ERROR_MISSING, \
    INSTALL_ERROR_MISSING_BUILD_FOLDER
from conans.client.source import complete_recipe_sources, config_source
from conans.client.tools.env import pythonpath
from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod,
                           conanfile_exception_formatter)
from conans.model.build_info import CppInfo
from conans.model.conan_file import get_env_context_manager
from conans.model.editable_layout import EditableLayout
from conans.model.env_info import EnvInfo
from conans.model.graph_info import GraphInfo
from conans.model.info import PACKAGE_ID_UNKNOWN
from conans.model.manifest import FileTreeManifest
from conans.model.ref import PackageReference
from conans.model.user_info import UserInfo
from conans.paths import BUILD_INFO, CONANINFO, RUN_LOG_NAME
from conans.util.env_reader import get_env
from conans.util.files import (clean_dirty, is_dirty, make_read_only, mkdir, rmdir, save, set_dirty,
                               set_dirty_context_manager)
from conans.util.log import logger
from conans.util.tracer import log_package_built, log_package_got_from_local_cache


def build_id(conan_file):
    if hasattr(conan_file, "build_id"):
        # construct new ConanInfo
        build_id_info = conan_file.info.copy()
        conan_file.info_build = build_id_info
        # effectively call the user function to change the package values
        with conanfile_exception_formatter(str(conan_file), "build_id"):
            conan_file.build_id()
        # compute modified ID
        return build_id_info.package_id()
    return None


class _PackageBuilder(object):
    def __init__(self, cache, output, hook_manager, remote_manager):
        self._cache = cache
        self._output = output
        self._hook_manager = hook_manager
        self._remote_manager = remote_manager

    def _get_build_folder(self, conanfile, package_layout, pref, keep_build, recorder):
        # Build folder can use a different package_ID if build_id() is defined.
        # This function decides if the build folder should be re-used (not build again)
        # and returns the build folder
        new_id = build_id(conanfile)
        build_pref = PackageReference(pref.ref, new_id) if new_id else pref
        build_folder = package_layout.build(build_pref)

        if is_dirty(build_folder):
            self._output.warn("Build folder is dirty, removing it: %s" % build_folder)
            rmdir(build_folder)

        # Decide if the build folder should be kept
        skip_build = conanfile.develop and keep_build
        if skip_build:
            self._output.info("Won't be built as specified by --keep-build")
            if not os.path.exists(build_folder):
                msg = "--keep-build specified, but build folder not found"
                recorder.package_install_error(pref, INSTALL_ERROR_MISSING_BUILD_FOLDER,
                                               msg, remote_name=None)
                raise ConanException(msg)
        elif build_pref != pref and os.path.exists(build_folder) and hasattr(conanfile, "build_id"):
            self._output.info("Won't be built, using previous build folder as defined in build_id()")
            skip_build = True

        return build_folder, skip_build

    def _prepare_sources(self, conanfile, pref, package_layout, conanfile_path, source_folder,
                         build_folder, remotes):
        export_folder = package_layout.export()
        export_source_folder = package_layout.export_sources()
        scm_sources_folder = package_layout.scm_sources()

        complete_recipe_sources(self._remote_manager, self._cache, conanfile, pref.ref, remotes)
        _remove_folder_raising(build_folder)

        config_source(export_folder, export_source_folder, scm_sources_folder, source_folder,
                      conanfile, self._output, conanfile_path, pref.ref,
                      self._hook_manager, self._cache)

        if not getattr(conanfile, 'no_copy_source', False):
            self._output.info('Copying sources to build folder')
            try:
                shutil.copytree(source_folder, build_folder, symlinks=True)
            except Exception as e:
                msg = str(e)
                if "206" in msg:  # System error shutil.Error 206: Filename or extension too long
                    msg += "\nUse short_paths=True if paths too long"
                raise ConanException("%s\nError copying sources to build folder" % msg)
            logger.debug("BUILD: Copied to %s", build_folder)
            logger.debug("BUILD: Files copied %s", ",".join(os.listdir(build_folder)))

    def _build(self, conanfile, pref, build_folder):
        # Read generators from conanfile and generate the needed files
        logger.info("GENERATORS: Writing generators")
        write_generators(conanfile, build_folder, self._output)

        # Build step might need DLLs, binaries as protoc to generate source files
        # So execute imports() before build, storing the list of copied_files
        copied_files = run_imports(conanfile, build_folder)

        try:
            self._hook_manager.execute("pre_build", conanfile=conanfile,
                                       reference=pref.ref, package_id=pref.id)
            logger.debug("Call conanfile.build() with files in build folder: %s",
                         os.listdir(build_folder))
            self._output.highlight("Calling build()")
            with conanfile_exception_formatter(str(conanfile), "build"):
                conanfile.build()

            self._output.success("Package '%s' built" % pref.id)
            self._output.info("Build folder %s" % build_folder)
            self._hook_manager.execute("post_build", conanfile=conanfile,
                                       reference=pref.ref, package_id=pref.id)
        except Exception as exc:
            self._output.writeln("")
            self._output.error("Package '%s' build failed" % pref.id)
            self._output.warn("Build folder %s" % build_folder)
            if isinstance(exc, ConanExceptionInUserConanfileMethod):
                raise exc
            raise ConanException(exc)
        finally:
            # Now remove all files that were imported with imports()
            remove_imports(conanfile, copied_files, self._output)

    def _package(self, conanfile, pref, package_layout, conanfile_path, build_folder,
                 package_folder):

        # FIXME: Is weak to assign here the recipe_hash
        manifest = package_layout.recipe_manifest()
        conanfile.info.recipe_hash = manifest.summary_hash

        # Creating ***info.txt files
        save(os.path.join(build_folder, CONANINFO), conanfile.info.dumps())
        self._output.info("Generated %s" % CONANINFO)
        save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conanfile).content)
        self._output.info("Generated %s" % BUILD_INFO)

        package_id = pref.id
        # Do the actual copy, call the conanfile.package() method
        with get_env_context_manager(conanfile):
            # Could be source or build depends no_copy_source
            source_folder = conanfile.source_folder
            install_folder = build_folder  # While installing, the infos goes to build folder
            prev = run_package_method(conanfile, package_id, source_folder, build_folder,
                                      package_folder, install_folder, self._hook_manager,
                                      conanfile_path, pref.ref)

        update_package_metadata(prev, package_layout, package_id, pref.ref.revision)

        if get_env("CONAN_READ_ONLY_CACHE", False):
            make_read_only(package_folder)
        # FIXME: Conan 2.0 Clear the registry entry (package ref)
        return prev

    def build_package(self, node, keep_build, recorder, remotes):
        t1 = time.time()

        conanfile = node.conanfile
        pref = node.pref

        package_layout = self._cache.package_layout(pref.ref, conanfile.short_paths)
        source_folder = package_layout.source()
        conanfile_path = package_layout.conanfile()
        package_folder = package_layout.package(pref)

        build_folder, skip_build = self._get_build_folder(conanfile, package_layout,
                                                          pref, keep_build, recorder)
        # PREPARE SOURCES
        if not skip_build:
            with package_layout.conanfile_write_lock(self._output):
                set_dirty(build_folder)
                self._prepare_sources(conanfile, pref, package_layout, conanfile_path, source_folder,
                                      build_folder, remotes)

        # BUILD & PACKAGE
        with package_layout.conanfile_read_lock(self._output):
            _remove_folder_raising(package_folder)
            mkdir(build_folder)
            with tools.chdir(build_folder):
                self._output.info('Building your package in %s' % build_folder)
                try:
                    if getattr(conanfile, 'no_copy_source', False):
                        conanfile.source_folder = source_folder
                    else:
                        conanfile.source_folder = build_folder

                    if not skip_build:
                        with get_env_context_manager(conanfile):
                            conanfile.build_folder = build_folder
                            conanfile.package_folder = package_folder
                            # In local cache, install folder always is build_folder
                            conanfile.install_folder = build_folder
                            self._build(conanfile, pref, build_folder)
                        clean_dirty(build_folder)

                    prev = self._package(conanfile, pref, package_layout, conanfile_path, build_folder,
                                         package_folder)
                    assert prev
                    node.prev = prev
                    log_file = os.path.join(build_folder, RUN_LOG_NAME)
                    log_file = log_file if os.path.exists(log_file) else None
                    log_package_built(pref, time.time() - t1, log_file)
                    recorder.package_built(pref)
                except ConanException as exc:
                    recorder.package_install_error(pref, INSTALL_ERROR_BUILDING,
                                                   str(exc), remote_name=None)
                    raise exc

            return node.pref


def _remove_folder_raising(folder):
    try:
        rmdir(folder)
    except OSError as e:
        raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n"
                             "Close any app using it, and retry" % str(e))


def _handle_system_requirements(conan_file, pref, cache, out):
    """ check first the system_reqs/system_requirements.txt existence, if not existing
    check package/sha1/

    Used after remote package retrieving and before package building
    """
    if "system_requirements" not in type(conan_file).__dict__:
        return

    package_layout = cache.package_layout(pref.ref)
    system_reqs_path = package_layout.system_reqs()
    system_reqs_package_path = package_layout.system_reqs_package(pref)
    if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path):
        return

    ret = call_system_requirements(conan_file, out)

    try:
        ret = str(ret or "")
    except Exception:
        out.warn("System requirements didn't return a string")
        ret = ""
    if getattr(conan_file, "global_system_requirements", None):
        save(system_reqs_path, ret)
    else:
        save(system_reqs_package_path, ret)


def call_system_requirements(conanfile, output):
    try:
        return conanfile.system_requirements()
    except Exception as e:
        output.error("while executing system_requirements(): %s" % str(e))
        raise ConanException("Error in system requirements")


def raise_package_not_found_error(conan_file, ref, package_id, dependencies, out, recorder):
    settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines())
    options_text = ", ".join(conan_file.info.full_options.dumps().splitlines())
    dependencies_text = ', '.join(dependencies)

    msg = '''Can't find a '%s' package for the specified settings, options and dependencies:
- Settings: %s
- Options: %s
- Dependencies: %s
- Package ID: %s
''' % (ref, settings_text, options_text, dependencies_text, package_id)
    out.warn(msg)
    recorder.package_install_error(PackageReference(ref, package_id), INSTALL_ERROR_MISSING, msg)
    raise ConanException('''Missing prebuilt package for '%s'
Try to build it from sources with "--build %s"
Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package"
''' % (ref, ref.name))


class BinaryInstaller(object):
    """ main responsible of retrieving binary packages or building them from source
    locally in case they are not found in remotes
    """
    def __init__(self, app, recorder):
        self._cache = app.cache
        self._out = app.out
        self._remote_manager = app.remote_manager
        self._recorder = recorder
        self._binaries_analyzer = app.binaries_analyzer
        self._hook_manager = app.hook_manager

    def install(self, deps_graph, remotes, build_mode, update, keep_build=False, graph_info=None):
        # order by levels and separate the root node (ref=None) from the rest
        nodes_by_level = deps_graph.by_levels()
        root_level = nodes_by_level.pop()
        root_node = root_level[0]
        # Get the nodes in order and if we have to build them
        self._build(nodes_by_level, keep_build, root_node, graph_info, remotes, build_mode, update)

    def _build(self, nodes_by_level, keep_build, root_node, graph_info, remotes, build_mode, update):
        processed_package_refs = set()
        for level in nodes_by_level:
            for node in level:
                ref, conan_file = node.ref, node.conanfile
                output = conan_file.output
                package_id = node.package_id
                if node.binary == BINARY_MISSING:
                    dependencies = [str(dep.dst) for dep in node.dependencies]
                    raise_package_not_found_error(conan_file, ref, package_id, dependencies,
                                                  out=output, recorder=self._recorder)

                self._propagate_info(node)
                if node.binary == BINARY_EDITABLE:
                    self._handle_node_editable(node, graph_info)
                else:
                    if node.binary == BINARY_SKIP:  # Privates not necessary
                        continue
                    assert ref.revision is not None, "Installer should receive RREV always"
                    _handle_system_requirements(conan_file, node.pref, self._cache, output)
                    if node.binary == BINARY_UNKNOWN:
                        self._binaries_analyzer.reevaluate_node(node, remotes, build_mode, update)
                    self._handle_node_cache(node, keep_build, processed_package_refs, remotes)

        # Finally, propagate information to root node (ref=None)
        self._propagate_info(root_node)

    @staticmethod
    def _node_concurrently_installed(node, package_folder):
        if node.binary == BINARY_DOWNLOAD and os.path.exists(package_folder):
            return True
        elif node.binary == BINARY_UPDATE:
            read_manifest = FileTreeManifest.load(package_folder)
            if node.update_manifest == read_manifest:
                return True

    def _handle_node_editable(self, node, graph_info):
        # Get source of information
        package_layout = self._cache.package_layout(node.ref)
        base_path = package_layout.base_folder()
        self._call_package_info(node.conanfile, package_folder=base_path, ref=node.ref)

        node.conanfile.cpp_info.filter_empty = False
        # Try with package-provided file
        editable_cpp_info = package_layout.editable_cpp_info()
        if editable_cpp_info:
            editable_cpp_info.apply_to(node.ref,
                                       node.conanfile.cpp_info,
                                       settings=node.conanfile.settings,
                                       options=node.conanfile.options)

            build_folder = editable_cpp_info.folder(node.ref, EditableLayout.BUILD_FOLDER,
                                                    settings=node.conanfile.settings,
                                                    options=node.conanfile.options)
            if build_folder is not None:
                build_folder = os.path.join(base_path, build_folder)
                output = node.conanfile.output
                write_generators(node.conanfile, build_folder, output)
                save(os.path.join(build_folder, CONANINFO), node.conanfile.info.dumps())
                output.info("Generated %s" % CONANINFO)
                graph_info_node = GraphInfo(graph_info.profile_host, root_ref=node.ref)
                graph_info_node.options = node.conanfile.options.values
                graph_info_node.graph_lock = graph_info.graph_lock
                graph_info_node.save(build_folder)
                output.info("Generated graphinfo")
                save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(node.conanfile).content)
                output.info("Generated %s" % BUILD_INFO)
                # Build step might need DLLs, binaries as protoc to generate source files
                # So execute imports() before build, storing the list of copied_files
                copied_files = run_imports(node.conanfile, build_folder)
                report_copied_files(copied_files, output)

    def _handle_node_cache(self, node, keep_build, processed_package_references, remotes):
        pref = node.pref
        assert pref.id, "Package-ID without value"
        assert pref.id != PACKAGE_ID_UNKNOWN, "Package-ID error: %s" % str(pref)
        conanfile = node.conanfile
        output = conanfile.output

        layout = self._cache.package_layout(pref.ref, conanfile.short_paths)
        package_folder = layout.package(pref)

        with layout.package_lock(pref):
            if pref not in processed_package_references:
                processed_package_references.add(pref)
                if node.binary == BINARY_BUILD:
                    assert node.prev is None, "PREV for %s to be built should be None" % str(pref)
                    with set_dirty_context_manager(package_folder):
                        pref = self._build_package(node, output, keep_build, remotes)
                    assert node.prev, "Node PREV shouldn't be empty"
                    assert node.pref.revision, "Node PREF revision shouldn't be empty"
                    assert pref.revision is not None, "PREV for %s to be built is None" % str(pref)
                elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD):
                    assert node.prev, "PREV for %s is None" % str(pref)
                    # not really concurrently, but a different node with same pref
                    if not self._node_concurrently_installed(node, package_folder):
                        with set_dirty_context_manager(package_folder):
                            assert pref.revision is not None, \
                                "Installer should receive #PREV always"
                            self._remote_manager.get_package(pref, package_folder,
                                                             node.binary_remote, output,
                                                             self._recorder)
                            output.info("Downloaded package revision %s" % pref.revision)
                            with layout.update_metadata() as metadata:
                                metadata.packages[pref.id].remote = node.binary_remote.name
                    else:
                        output.success('Download skipped. Probable concurrent download')
                        log_package_got_from_local_cache(pref)
                        self._recorder.package_fetched_from_cache(pref)
                elif node.binary == BINARY_CACHE:
                    assert node.prev, "PREV for %s is None" % str(pref)
                    output.success('Already installed!')
                    log_package_got_from_local_cache(pref)
                    self._recorder.package_fetched_from_cache(pref)

            # Call the info method
            self._call_package_info(conanfile, package_folder, ref=pref.ref)
            self._recorder.package_cpp_info(pref, conanfile.cpp_info)

    def _build_package(self, node, output, keep_build, remotes):
        conanfile = node.conanfile
        # It is necessary to complete the sources of python requires, which might be used
        for python_require in conanfile.python_requires.values():
            assert python_require.ref.revision is not None, \
                "Installer should receive python_require.ref always"
            complete_recipe_sources(self._remote_manager, self._cache,
                                    python_require.conanfile, python_require.ref, remotes)

        builder = _PackageBuilder(self._cache, output, self._hook_manager, self._remote_manager)
        pref = builder.build_package(node, keep_build, self._recorder, remotes)
        if node.graph_lock_node:
            node.graph_lock_node.modified = BINARY_BUILD
        return pref

    @staticmethod
    def _propagate_info(node):
        # Get deps_cpp_info from upstream nodes
        node_order = [n for n in node.public_closure if n.binary != BINARY_SKIP]
        # List sort is stable, will keep the original order of the closure, but prioritize levels
        conan_file = node.conanfile
        transitive = node.transitive_closure.values()
        for n in node_order:
            if n not in transitive:
                conan_file.output.info("Applying build-requirement: %s" % str(n.ref))
            conan_file.deps_cpp_info.update(n.conanfile.cpp_info, n.ref.name)
            conan_file.deps_env_info.update(n.conanfile.env_info, n.ref.name)
            conan_file.deps_user_info[n.ref.name] = n.conanfile.user_info

        # Update the info but filtering the package values that not apply to the subtree
        # of this current node and its dependencies.
        subtree_libnames = [node.ref.name for node in node_order]
        for package_name, env_vars in conan_file._conan_env_values.data.items():
            for name, value in env_vars.items():
                if not package_name or package_name in subtree_libnames or \
                   package_name == conan_file.name:
                    conan_file.info.env_values.add(name, value, package_name)

    def _call_package_info(self, conanfile, package_folder, ref):
        conanfile.cpp_info = CppInfo(package_folder)
        conanfile.cpp_info.name = conanfile.name
        conanfile.cpp_info.version = conanfile.version
        conanfile.cpp_info.description = conanfile.description
        conanfile.env_info = EnvInfo()
        conanfile.user_info = UserInfo()

        # Get deps_cpp_info from upstream nodes
        public_deps = [name for name, req in conanfile.requires.items() if not req.private
                       and not req.override]
        conanfile.cpp_info.public_deps = public_deps
        # Once the node is build, execute package info, so it has access to the
        # package folder and artifacts
        with pythonpath(conanfile):  # Minimal pythonpath, not the whole context, make it 50% slower
            with tools.chdir(package_folder):
                with conanfile_exception_formatter(str(conanfile), "package_info"):
                    conanfile.package_folder = package_folder
                    conanfile.source_folder = None
                    conanfile.build_folder = None
                    conanfile.install_folder = None
                    self._hook_manager.execute("pre_package_info", conanfile=conanfile,
                                               reference=ref)
                    conanfile.package_info()
                    self._hook_manager.execute("post_package_info", conanfile=conanfile,
                                               reference=ref)