graph_manager.py

import fnmatch
import os
from collections import OrderedDict, defaultdict

from conans.client.conanfile.configure import run_configure_method
from conans.client.generators.text import TXTGenerator
from conans.client.graph.build_mode import BuildMode
from conans.client.graph.graph import BINARY_BUILD, Node, CONTEXT_HOST, CONTEXT_BUILD
from conans.client.graph.graph_binaries import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_EDITABLE, \
    BINARY_UNKNOWN
from conans.client.graph.graph_builder import DepsGraphBuilder
from conans.errors import ConanException, conanfile_exception_formatter, ConanInvalidConfiguration
from conans.model.conan_file import get_env_context_manager
from conans.model.graph_info import GraphInfo
from conans.model.graph_lock import GraphLock, GraphLockFile
from conans.model.ref import ConanFileReference
from conans.paths import BUILD_INFO
from conans.util.files import load


class _RecipeBuildRequires(OrderedDict):
    def __init__(self, conanfile, default_context):
        super(_RecipeBuildRequires, self).__init__()
        # "tool_requires" is an alias for 2.0 compatibility
        for require_type in ("build_requires", "tool_requires"):
            build_requires = getattr(conanfile, require_type, [])
            if not isinstance(build_requires, (list, tuple)):
                build_requires = [build_requires]
            self._default_context = default_context
            for build_require in build_requires:
                self.add(build_require, context=self._default_context)

    def add(self, build_require, context, force_host_context=False):
        if build_require is None:
            return
        if not isinstance(build_require, ConanFileReference):
            build_require = ConanFileReference.loads(build_require)
        build_require.force_host_context = force_host_context  # Dirty, but will be removed in 2.0
        self[(build_require.name, context)] = build_require

    def __call__(self, build_require, force_host_context=False):
        context = CONTEXT_HOST if force_host_context else self._default_context
        self.add(build_require, context, force_host_context)

    def __str__(self):
        items = ["{} ({})".format(br, ctxt) for (_, ctxt), br in self.items()]
        return ", ".join(items)


class GraphManager(object):
    def __init__(self, output, cache, remote_manager, loader, proxy, resolver, binary_analyzer):
        self._proxy = proxy
        self._output = output
        self._resolver = resolver
        self._cache = cache
        self._remote_manager = remote_manager
        self._loader = loader
        self._binary_analyzer = binary_analyzer

    def load_consumer_conanfile(self, conanfile_path, info_folder,
                                deps_info_required=False, test=False):
        """loads a conanfile for local flow: source, imports, package, build
        """
        try:
            graph_info = GraphInfo.load(info_folder)
            lock_path = os.path.join(info_folder, "conan.lock")
            graph_lock_file = GraphLockFile.load(lock_path, self._cache.config.revisions_enabled)
            graph_lock = graph_lock_file.graph_lock
            self._output.info("Using lockfile: '{}/conan.lock'".format(info_folder))
            profile_host = graph_lock_file.profile_host
            profile_build = graph_lock_file.profile_build
            self._output.info("Using cached profile from lockfile")
        except IOError:  # Only if file is missing
            graph_lock = None
            # This is very dirty, should be removed for Conan 2.0 (source() method only)
            profile_host = self._cache.default_profile
            profile_host.process_settings(self._cache)
            profile_build = None
            name, version, user, channel = None, None, None, None
        else:
            name, version, user, channel, _ = graph_info.root
            profile_host.process_settings(self._cache, preprocess=False)
            # This is the hack of recovering the options from the graph_info
            profile_host.options.update(graph_info.options)
            if profile_build:
                profile_build.process_settings(self._cache, preprocess=False)
        if conanfile_path.endswith(".py"):
            lock_python_requires = None
            if graph_lock and not test:  # Only lock python requires if it is not test_package
                node_id = graph_lock.get_consumer(graph_info.root)
                lock_python_requires = graph_lock.python_requires(node_id)
            # The global.conf is necessary for download_cache definition
            profile_host.conf.rebase_conf_definition(self._cache.new_config)
            conanfile = self._loader.load_consumer(conanfile_path,
                                                   profile_host=profile_host,
                                                   name=name, version=version,
                                                   user=user, channel=channel,
                                                   lock_python_requires=lock_python_requires)

            if profile_build:
                conanfile.settings_build = profile_build.processed_settings.copy()
                conanfile.settings_target = None

            if test:
                conanfile.display_name = "%s (test package)" % str(test)
                conanfile.output.scope = conanfile.display_name
                conanfile.tested_reference_str = repr(test)
            run_configure_method(conanfile, down_options=None, down_ref=None, ref=None)
        else:
            conanfile = self._loader.load_conanfile_txt(conanfile_path, profile_host=profile_host)

        load_deps_info(info_folder, conanfile, required=deps_info_required)

        return conanfile

    def load_graph(self, reference, create_reference, graph_info, build_mode, check_updates, update,
                   remotes, recorder, apply_build_requires=True, lockfile_node_id=None,
                   is_build_require=False, require_overrides=None):
        """ main entry point to compute a full dependency graph
        """
        profile_host, profile_build = graph_info.profile_host, graph_info.profile_build
        graph_lock, root_ref = graph_info.graph_lock, graph_info.root

        root_node = self._load_root_node(reference, create_reference, profile_host, graph_lock,
                                         root_ref, lockfile_node_id, is_build_require,
                                         require_overrides)

        deps_graph = self._resolve_graph(root_node, profile_host, profile_build, graph_lock,
                                         build_mode, check_updates, update, remotes, recorder,
                                         apply_build_requires=apply_build_requires)
        # Run some validations once the graph is built
        self._validate_graph_provides(deps_graph)

        # THIS IS NECESSARY to store dependencies options in profile, for consumer
        # FIXME: This is a hack. Might dissapear if graph for local commands is always recomputed
        graph_info.options = root_node.conanfile.options.values
        if root_node.ref:
            graph_info.root = root_node.ref

        if graph_info.graph_lock is None:
            graph_info.graph_lock = GraphLock(deps_graph, self._cache.config.revisions_enabled)

        return deps_graph

    def _load_root_node(self, reference, create_reference, profile_host, graph_lock, root_ref,
                        lockfile_node_id, is_build_require, require_overrides):
        """ creates the first, root node of the graph, loading or creating a conanfile
        and initializing it (settings, options) as necessary. Also locking with lockfile
        information
        """
        profile_host.dev_reference = create_reference  # Make sure the created one has develop=True

        if isinstance(reference, list):  # Install workspace with multiple root nodes
            conanfile = self._loader.load_virtual(reference, profile_host, scope_options=False)
            # Locking in workspaces not implemented yet
            return Node(ref=None, context=CONTEXT_HOST, conanfile=conanfile, recipe=RECIPE_VIRTUAL)

        # create (without test_package), install|info|graph|export-pkg <ref>
        if isinstance(reference, ConanFileReference):
            return self._load_root_direct_reference(reference, graph_lock, profile_host,
                                                    lockfile_node_id, is_build_require,
                                                    require_overrides)

        path = reference  # The reference must be pointing to a user space conanfile
        if create_reference:  # Test_package -> tested reference
            ret = self._load_root_test_package(path, create_reference, graph_lock, profile_host,
                                                require_overrides)
            return ret

        # It is a path to conanfile.py or conanfile.txt
        root_node = self._load_root_consumer(path, graph_lock, profile_host, root_ref,
                                             require_overrides)
        return root_node

    def _load_root_consumer(self, path, graph_lock, profile, ref, require_overrides):
        """ load a CONSUMER node from a user space conanfile.py or conanfile.txt
        install|info|create|graph <path>
        :path full path to a conanfile
        :graph_lock: might be None, information of lockfiles
        :profile: data to inject to the consumer node: settings, options
        :ref: previous reference of a previous command. Can be used for finding itself in
              the lockfile, or to initialize
        """
        if path.endswith(".py"):
            lock_python_requires = None
            if graph_lock:
                if ref.name is None:
                    # If the graph_info information is not there, better get what we can from
                    # the conanfile
                    # Using load_named() to run set_name() set_version() and get them
                    # so it can be found by name in the lockfile
                    conanfile = self._loader.load_named(path, None, None, None, None)
                    ref = ConanFileReference(ref.name or conanfile.name,
                                             ref.version or conanfile.version,
                                             ref.user, ref.channel, validate=False)
                node_id = graph_lock.get_consumer(ref)
                lock_python_requires = graph_lock.python_requires(node_id)

            conanfile = self._loader.load_consumer(path, profile,
                                                   name=ref.name,
                                                   version=ref.version,
                                                   user=ref.user,
                                                   channel=ref.channel,
                                                   lock_python_requires=lock_python_requires,
                                                   require_overrides=require_overrides)

            ref = ConanFileReference(conanfile.name, conanfile.version,
                                     ref.user, ref.channel, validate=False)
            root_node = Node(ref, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER, path=path)
        else:
            conanfile = self._loader.load_conanfile_txt(path, profile, ref=ref,
                                                        require_overrides=require_overrides)
            root_node = Node(None, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER,
                             path=path)

        if graph_lock:  # Find the Node ID in the lock of current root
            node_id = graph_lock.get_consumer(root_node.ref)
            root_node.id = node_id

        return root_node

    def _load_root_direct_reference(self, reference, graph_lock, profile, lockfile_node_id,
                                    is_build_require, require_overrides):
        """ When a full reference is provided:
        install|info|graph <ref> or export-pkg .
        :return a VIRTUAL root_node with a conanfile that requires the reference
        """
        if not self._cache.config.revisions_enabled and reference.revision is not None:
            raise ConanException("Revisions not enabled in the client, specify a "
                                 "reference without revision")

        conanfile = self._loader.load_virtual([reference], profile,
                                              is_build_require=is_build_require,
                                              require_overrides=require_overrides)
        root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL)
        # Build_requires cannot be found as early as this, because there is no require yet
        if graph_lock and not is_build_require:  # Find the Node ID in the lock of current root
            graph_lock.find_require_and_lock(reference, conanfile, lockfile_node_id)
        return root_node

    def _load_root_test_package(self, path, create_reference, graph_lock, profile,
                                require_overrides):
        """ when a test_package/conanfile.py is provided, together with the reference that is
        being created and need to be tested
        :return a CONSUMER root_node with a conanfile.py with an injected requires to the
        created reference
        """
        test = str(create_reference)
        # do not try apply lock_python_requires for test_package/conanfile.py consumer
        conanfile = self._loader.load_consumer(path, profile, user=create_reference.user,
                                               channel=create_reference.channel,
                                               require_overrides=require_overrides
                                               )
        conanfile.display_name = "%s (test package)" % str(test)
        conanfile.output.scope = conanfile.display_name
        conanfile.tested_reference_str = repr(create_reference)

        # Injection of the tested reference
        test_type = getattr(conanfile, "test_type", ("requires", ))
        if not isinstance(test_type, (list, tuple)):
            test_type = (test_type, )

        if "explicit" not in test_type:  # 2.0 mode, not automatically add the require, always explicit
            if "build_requires" in test_type:
                if getattr(conanfile, "build_requires", None):
                    # Injecting the tested reference
                    existing = conanfile.build_requires
                    if not isinstance(existing, (list, tuple)):
                        existing = [existing]
                    conanfile.build_requires = list(existing) + [create_reference]
                else:
                    conanfile.build_requires = str(create_reference)
            if "requires" in test_type:
                require = conanfile.requires.get(create_reference.name)
                if require:
                    require.ref = require.range_ref = create_reference
                else:
                    conanfile.requires.add_ref(create_reference)

        ref = ConanFileReference(conanfile.name, conanfile.version,
                                 create_reference.user, create_reference.channel, validate=False)
        root_node = Node(ref, conanfile, recipe=RECIPE_CONSUMER, context=CONTEXT_HOST, path=path)
        if graph_lock:
            graph_lock.find_require_and_lock(create_reference, conanfile)
        return root_node

    def _resolve_graph(self, root_node, profile_host, profile_build, graph_lock, build_mode,
                       check_updates, update, remotes, recorder, apply_build_requires=True):
        build_mode = BuildMode(build_mode, self._output)
        deps_graph = self._load_graph(root_node, check_updates, update,
                                      build_mode=build_mode, remotes=remotes,
                                      recorder=recorder,
                                      profile_host=profile_host,
                                      profile_build=profile_build,
                                      apply_build_requires=apply_build_requires,
                                      graph_lock=graph_lock)

        version_ranges_output = self._resolver.output
        if version_ranges_output:
            self._output.success("Version ranges solved")
            for msg in version_ranges_output:
                self._output.info("    %s" % msg)
            self._output.writeln("")
            self._resolver.clear_output()

        build_mode.report_matches()
        return deps_graph

    @staticmethod
    def _get_recipe_build_requires(conanfile, default_context):
        conanfile.build_requires = _RecipeBuildRequires(conanfile, default_context)
        conanfile.tool_requires = conanfile.build_requires

        class TestRequirements:
            def __init__(self, build_requires):
                self._build_requires = build_requires

            def __call__(self, ref):
                self._build_requires(ref, force_host_context=True)

        conanfile.test_requires = TestRequirements(conanfile.build_requires)
        if hasattr(conanfile, "build_requirements"):
            with get_env_context_manager(conanfile):
                with conanfile_exception_formatter(str(conanfile), "build_requirements"):
                    conanfile.build_requirements()

        return conanfile.build_requires

    def _recurse_build_requires(self, graph, builder, check_updates,
                                update, build_mode, remotes, profile_build_requires, recorder,
                                profile_host, profile_build, graph_lock, apply_build_requires=True,
                                nodes_subset=None, root=None):
        """
        :param graph: This is the full dependency graph with all nodes from all recursions
        """
        default_context = CONTEXT_BUILD if profile_build else CONTEXT_HOST
        self._binary_analyzer.evaluate_graph(graph, build_mode, update, remotes, nodes_subset, root)
        if not apply_build_requires:
            return

        for node in graph.ordered_iterate(nodes_subset):
            # Packages with PACKAGE_ID_UNKNOWN might be built in the future, need build requires
            if (node.binary not in (BINARY_BUILD, BINARY_EDITABLE, BINARY_UNKNOWN)
                    and node.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL)):
                continue
            package_build_requires = self._get_recipe_build_requires(node.conanfile, default_context)
            str_ref = str(node.ref)

            #  Compute the update of the current recipe build_requires when updated with the
            # downstream profile-defined build-requires
            new_profile_build_requires = []
            for pattern, build_requires in profile_build_requires.items():
                if node.recipe == RECIPE_VIRTUAL:  # Virtual do not get profile build requires
                    continue
                if ((node.recipe == RECIPE_CONSUMER and pattern == "&") or
                        (node.recipe != RECIPE_CONSUMER and pattern == "&!") or
                        fnmatch.fnmatch(str_ref, pattern)):
                    for build_require in build_requires:
                        br_key = (build_require.name, default_context)
                        if br_key in package_build_requires:  # Override defined
                            # this is a way to have only one package Name for all versions
                            # (no conflicts)
                            # but the dict key is not used at all
                            package_build_requires[br_key] = build_require
                        # Profile one or in different context
                        elif build_require.name != node.name or default_context != node.context:
                            new_profile_build_requires.append((build_require, default_context))

            def _recurse_build_requires(br_list, transitive_build_requires):
                nodessub = builder.extend_build_requires(graph, node, br_list, check_updates,
                                                         update, remotes, profile_host,
                                                         profile_build, graph_lock)
                self._recurse_build_requires(graph, builder, check_updates, update, build_mode,
                                             remotes, transitive_build_requires, recorder,
                                             profile_host, profile_build, graph_lock,
                                             nodes_subset=nodessub, root=node)

            if package_build_requires:
                if default_context == CONTEXT_BUILD:
                    br_build, br_host = [], []
                    for (_, ctxt), it in package_build_requires.items():
                        if ctxt == CONTEXT_BUILD:
                            br_build.append((it, ctxt))
                        else:
                            br_host.append((it, ctxt))
                    if br_build:
                        _recurse_build_requires(br_build, profile_build.build_requires)
                    if br_host:
                        _recurse_build_requires(br_host, profile_build_requires)
                else:
                    br_all = [(it, ctxt) for (_, ctxt), it in package_build_requires.items()]
                    _recurse_build_requires(br_all, profile_build_requires)

            if new_profile_build_requires:
                _recurse_build_requires(new_profile_build_requires, {})

            if graph_lock and node.recipe != RECIPE_VIRTUAL:
                graph_lock.check_locked_build_requires(node, package_build_requires,
                                                       new_profile_build_requires)

    def _load_graph(self, root_node, check_updates, update, build_mode, remotes,
                    recorder, profile_host, profile_build, apply_build_requires,
                    graph_lock):
        assert isinstance(build_mode, BuildMode)
        profile_host_build_requires = profile_host.build_requires
        builder = DepsGraphBuilder(self._proxy, self._output, self._loader, self._resolver,
                                   recorder)
        graph = builder.load_graph(root_node, check_updates, update, remotes, profile_host,
                                   profile_build, graph_lock)

        self._recurse_build_requires(graph, builder, check_updates, update, build_mode,
                                     remotes, profile_host_build_requires, recorder, profile_host,
                                     profile_build, graph_lock,
                                     apply_build_requires=apply_build_requires)

        # Sort of closures, for linking order
        inverse_levels = {n: i for i, level in enumerate(graph.inverse_levels()) for n in level}
        for node in graph.nodes:
            node.public_closure.pop(node.name, context=node.context)
            # List sort is stable, will keep the original order of closure, but prioritize levels
            node.public_closure.sort(key_fn=lambda n: inverse_levels[n])

        return graph

    @staticmethod
    def _validate_graph_provides(deps_graph):
        # Check that two different nodes are not providing the same (ODR violation)
        for node in deps_graph.nodes:
            provides = defaultdict(list)
            if node.conanfile.provides is not None:  # consumer conanfile doesn't initialize
                for it in node.conanfile.provides:
                    provides[it].append(node)

            for item in filter(lambda u: u.context == CONTEXT_HOST, node.public_closure):
                for it in item.conanfile.provides:
                    provides[it].append(item)

            # Check (and report) if any functionality is provided by several different recipes
            conflicts = [it for it, nodes in provides.items() if len(nodes) > 1]
            if conflicts:
                msg_lines = ["At least two recipes provides the same functionality:"]
                for it in conflicts:
                    nodes_str = "', '".join([n.conanfile.display_name for n in provides[it]])
                    msg_lines.append(" - '{}' provided by '{}'".format(it, nodes_str))
                raise ConanException('\n'.join(msg_lines))


def load_deps_info(current_path, conanfile, required):
    def get_forbidden_access_object(field_name):
        class InfoObjectNotDefined(object):
            def __getitem__(self, item):
                raise ConanException("self.%s not defined. If you need it for a "
                                     "local command run 'conan install'" % field_name)

            __getattr__ = __getitem__

        return InfoObjectNotDefined()

    if not current_path:
        return
    info_file_path = os.path.join(current_path, BUILD_INFO)
    try:
        deps_cpp_info, deps_user_info, deps_env_info, user_info_build = \
            TXTGenerator.loads(load(info_file_path), filter_empty=True)
        conanfile.deps_cpp_info = deps_cpp_info
        conanfile.deps_user_info = deps_user_info
        conanfile.deps_env_info = deps_env_info
        if user_info_build:
            conanfile.user_info_build = user_info_build
    except IOError:
        if required:
            raise ConanException("%s file not found in %s\nIt is required for this command\n"
                                 "You can generate it using 'conan install'"
                                 % (BUILD_INFO, current_path))
        conanfile.deps_cpp_info = get_forbidden_access_object("deps_cpp_info")
        conanfile.deps_user_info = get_forbidden_access_object("deps_user_info")
    except ConanException:
        raise ConanException("Parse error in '%s' file in %s" % (BUILD_INFO, current_path))