diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index e22ef85c9db..a70c7f08b6e 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -2,6 +2,7 @@ import sys from collections import OrderedDict +from conans.client.graph.graph_binaries import GraphBinariesAnalyzer from conans.client.manager import deps_install from conans.paths.package_layouts.package_cache_layout import PackageCacheLayout @@ -181,9 +182,9 @@ def __init__(self, cache_folder, user_io, http_requester=None, runner=None): self.python_requires = ConanPythonRequire(self.proxy, resolver) self.loader = ConanFileLoader(self.runner, self.out, self.python_requires) - self.graph_manager = GraphManager(self.out, self.cache, - self.remote_manager, self.loader, self.proxy, - resolver) + self.binaries_analyzer = GraphBinariesAnalyzer(self.cache, self.out, self.remote_manager) + self.graph_manager = GraphManager(self.out, self.cache, self.remote_manager, self.loader, + self.proxy, resolver, self.binaries_analyzer) def load_remotes(self, remote_name=None, update=False, check_updates=False): remotes = self.cache.registry.load_remotes() diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index 0b6758c50e8..0cfa9f1ca09 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -201,8 +201,6 @@ def __init__(self): self.nodes = set() self.root = None self.aliased = {} - # These are the nodes with pref (not including PREV) that have been evaluated - self.evaluated = {} # {pref: [nodes]} def add_node(self, node): if not self.nodes: diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index dac1ebcf2ab..1027aefa5f1 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -18,8 +18,11 @@ def __init__(self, cache, output, remote_manager): self._cache = cache self._out = output self._remote_manager = remote_manager + # These are the nodes with pref (not including PREV) that have been evaluated + self._evaluated = {} # {pref: [nodes]} - def _check_update(self, upstream_manifest, package_folder, output, node): + @staticmethod + def _check_update(upstream_manifest, package_folder, output, node): read_manifest = FileTreeManifest.load(package_folder) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: @@ -29,7 +32,126 @@ def _check_update(self, upstream_manifest, package_folder, output, node): else: output.warn("Current package is newer than remote upstream one") - def _evaluate_node(self, node, build_mode, update, evaluated_nodes, remotes): + @staticmethod + def _evaluate_build(node, build_mode): + ref, conanfile = node.ref, node.conanfile + with_deps_to_build = False + # For cascade mode, we need to check also the "modified" status of the lockfile if exists + # modified nodes have already been built, so they shouldn't be built again + if build_mode.cascade and not (node.graph_lock_node and node.graph_lock_node.modified): + for dep in node.dependencies: + dep_node = dep.dst + if (dep_node.binary == BINARY_BUILD or + (dep_node.graph_lock_node and dep_node.graph_lock_node.modified)): + with_deps_to_build = True + break + if build_mode.forced(conanfile, ref, with_deps_to_build): + conanfile.output.info('Forced build from source') + node.binary = BINARY_BUILD + node.prev = None + return True + + def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, package_folder, pref): + # Check if dirty, to remove it + with package_layout.package_lock(pref): + assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code" + if is_dirty(package_folder): + node.conanfile.output.warn("Package is corrupted, removing folder: %s" + % package_folder) + rmdir(package_folder) # Do not remove if it is EDITABLE + return + + if self._cache.config.revisions_enabled: + metadata = package_layout.load_metadata() + rec_rev = metadata.packages[pref.id].recipe_revision + if rec_rev and rec_rev != node.ref.revision: + node.conanfile.output.warn("The package {} doesn't belong to the installed " + "recipe revision, removing folder".format(pref)) + rmdir(package_folder) + return metadata + + def _evaluate_cache_pkg(self, node, package_layout, pref, metadata, remote, remotes, update, + package_folder): + if update: + output = node.conanfile.output + if remote: + try: + tmp = self._remote_manager.get_package_manifest(pref, remote) + upstream_manifest, pref = tmp + except NotFoundException: + output.warn("Can't update, no package in remote") + except NoRemoteAvailable: + output.warn("Can't update, no remote defined") + else: + if self._check_update(upstream_manifest, package_folder, output, node): + node.binary = BINARY_UPDATE + node.prev = pref.revision # With revision + elif remotes: + pass # Current behavior: no remote explicit or in metadata, do not update + else: + output.warn("Can't update, no remote defined") + if not node.binary: + node.binary = BINARY_CACHE + metadata = metadata or package_layout.load_metadata() + node.prev = metadata.packages[pref.id].revision + assert node.prev, "PREV for %s is None: %s" % (str(pref), metadata.dumps()) + + def _evaluate_remote_pkg(self, node, pref, remote, remotes, build_mode): + remote_info = None + if remote: + try: + remote_info, pref = self._remote_manager.get_package_info(pref, remote) + except NotFoundException: + pass + except Exception: + node.conanfile.output.error("Error downloading binary package: '{}'".format(pref)) + raise + + # If the "remote" came from the registry but the user didn't specified the -r, with + # revisions iterate all remotes + if not remote or (not remote_info and self._cache.config.revisions_enabled): + for r in remotes.values(): + try: + remote_info, pref = self._remote_manager.get_package_info(pref, r) + except NotFoundException: + pass + else: + if remote_info: + remote = r + break + + if remote_info: + node.binary = BINARY_DOWNLOAD + node.prev = pref.revision + recipe_hash = remote_info.recipe_hash + else: + recipe_hash = None + if build_mode.allowed(node.conanfile): + node.binary = BINARY_BUILD + else: + node.binary = BINARY_MISSING + node.prev = None + + return recipe_hash, remote + + def _evaluate_is_cached(self, node, pref): + previous_nodes = self._evaluated.get(pref) + if previous_nodes: + previous_nodes.append(node) + previous_node = previous_nodes[0] + # The previous node might have been skipped, but current one not necessarily + # keep the original node.binary value (before being skipped), and if it will be + # defined as SKIP again by self._handle_private(node) if it is really private + if previous_node.binary == BINARY_SKIP: + node.binary = previous_node.binary_non_skip + else: + node.binary = previous_node.binary + node.binary_remote = previous_node.binary_remote + node.prev = previous_node.prev + return True + self._evaluated[pref] = [node] + + def _evaluate_node(self, node, build_mode, update, remotes): assert node.binary is None, "Node.binary should be None" assert node.package_id is not None, "Node.package_id shouldn't be None" assert node.prev is None, "Node.prev should be None" @@ -48,145 +170,51 @@ def _evaluate_node(self, node, build_mode, update, evaluated_nodes, remotes): pref = PackageReference(ref, node.package_id) # Check that this same reference hasn't already been checked - previous_nodes = evaluated_nodes.get(pref) - if previous_nodes: - previous_nodes.append(node) - previous_node = previous_nodes[0] - # The previous node might have been skipped, but current one not necessarily - # keep the original node.binary value (before being skipped), and if it will be - # defined as SKIP again by self._handle_private(node) if it is really private - if previous_node.binary == BINARY_SKIP: - node.binary = previous_node.binary_non_skip - else: - node.binary = previous_node.binary - node.binary_remote = previous_node.binary_remote - node.prev = previous_node.prev + if self._evaluate_is_cached(node, pref): return - evaluated_nodes[pref] = [node] - - output = conanfile.output if node.recipe == RECIPE_EDITABLE: - node.binary = BINARY_EDITABLE - # TODO: PREV? + node.binary = BINARY_EDITABLE # TODO: PREV? return - with_deps_to_build = False - # For cascade mode, we need to check also the "modified" status of the lockfile if exists - # modified nodes have already been built, so they shouldn't be built again - if build_mode.cascade and not (node.graph_lock_node and node.graph_lock_node.modified): - for dep in node.dependencies: - dep_node = dep.dst - if (dep_node.binary == BINARY_BUILD or - (dep_node.graph_lock_node and dep_node.graph_lock_node.modified)): - with_deps_to_build = True - break - if build_mode.forced(conanfile, ref, with_deps_to_build): - output.info('Forced build from source') - node.binary = BINARY_BUILD - node.prev = None + if self._evaluate_build(node, build_mode): return - package_folder = self._cache.package_layout(pref.ref, - short_paths=conanfile.short_paths).package(pref) - - # Check if dirty, to remove it - with self._cache.package_layout(pref.ref).package_lock(pref): - assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code" - if is_dirty(package_folder): - output.warn("Package is corrupted, removing folder: %s" % package_folder) - rmdir(package_folder) # Do not remove if it is EDITABLE - - if self._cache.config.revisions_enabled: - metadata = self._cache.package_layout(pref.ref).load_metadata() - rec_rev = metadata.packages[pref.id].recipe_revision - if rec_rev and rec_rev != node.ref.revision: - output.warn("The package {} doesn't belong " - "to the installed recipe revision, removing folder".format(pref)) - rmdir(package_folder) + package_layout = self._cache.package_layout(pref.ref, short_paths=conanfile.short_paths) + package_folder = package_layout.package(pref) + metadata = self._evaluate_clean_pkg_folder_dirty(node, package_layout, package_folder, pref) remote = remotes.selected if not remote: - # If the remote_name is not given, follow the binary remote, or - # the recipe remote + # If the remote_name is not given, follow the binary remote, or the recipe remote # If it is defined it won't iterate (might change in conan2.0) - metadata = self._cache.package_layout(pref.ref).load_metadata() + metadata = metadata or package_layout.load_metadata() remote_name = metadata.packages[pref.id].remote or metadata.recipe.remote remote = remotes.get(remote_name) - if os.path.exists(package_folder): - if update: - if remote: - try: - tmp = self._remote_manager.get_package_manifest(pref, remote) - upstream_manifest, pref = tmp - except NotFoundException: - output.warn("Can't update, no package in remote") - except NoRemoteAvailable: - output.warn("Can't update, no remote defined") - else: - if self._check_update(upstream_manifest, package_folder, output, node): - node.binary = BINARY_UPDATE - node.prev = pref.revision # With revision - if build_mode.outdated: - info, pref = self._remote_manager.get_package_info(pref, remote) - package_hash = info.recipe_hash - elif remotes: - pass - else: - output.warn("Can't update, no remote defined") - if not node.binary: - node.binary = BINARY_CACHE - metadata = self._cache.package_layout(pref.ref).load_metadata() - node.prev = metadata.packages[pref.id].revision - assert node.prev, "PREV for %s is None: %s" % (str(pref), metadata.dumps()) - package_hash = ConanInfo.load_from_package(package_folder).recipe_hash - + if os.path.exists(package_folder): # Binary already in cache, check for updates + self._evaluate_cache_pkg(node, package_layout, pref, metadata, remote, remotes, update, + package_folder) + recipe_hash = None else: # Binary does NOT exist locally - remote_info = None - if remote: - try: - remote_info, pref = self._remote_manager.get_package_info(pref, remote) - except NotFoundException: - pass - except Exception: - conanfile.output.error("Error downloading binary package: '{}'".format(pref)) - raise - - # If the "remote" came from the registry but the user didn't specified the -r, with - # revisions iterate all remotes - - if not remote or (not remote_info and self._cache.config.revisions_enabled): - for r in remotes.values(): - try: - remote_info, pref = self._remote_manager.get_package_info(pref, r) - except NotFoundException: - pass - else: - if remote_info: - remote = r - break - - if remote_info: - node.binary = BINARY_DOWNLOAD - node.prev = pref.revision - package_hash = remote_info.recipe_hash - else: - if build_mode.allowed(conanfile): - node.binary = BINARY_BUILD - else: - node.binary = BINARY_MISSING - node.prev = None + # Returned remote might be different than the passed one if iterating remotes + recipe_hash, remote = self._evaluate_remote_pkg(node, pref, remote, remotes, build_mode) if build_mode.outdated: if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE): - local_recipe_hash = self._cache.package_layout(ref).recipe_manifest().summary_hash - if local_recipe_hash != package_hash: - output.info("Outdated package!") + if node.binary == BINARY_UPDATE: + info, pref = self._remote_manager.get_package_info(pref, remote) + recipe_hash = info.recipe_hash + elif node.binary == BINARY_CACHE: + recipe_hash = ConanInfo.load_from_package(package_folder).recipe_hash + + local_recipe_hash = package_layout.recipe_manifest().summary_hash + if local_recipe_hash != recipe_hash: + conanfile.output.info("Outdated package!") node.binary = BINARY_BUILD node.prev = None else: - output.info("Package is up to date") + conanfile.output.info("Package is up to date") node.binary_remote = remote @@ -241,10 +269,9 @@ def _handle_private(self, node): def evaluate_graph(self, deps_graph, build_mode, update, remotes): default_package_id_mode = self._cache.config.default_package_id_mode - evaluated = deps_graph.evaluated for node in deps_graph.ordered_iterate(): self._compute_package_id(node, default_package_id_mode) if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): continue - self._evaluate_node(node, build_mode, update, evaluated, remotes) + self._evaluate_node(node, build_mode, update, remotes) self._handle_private(node) diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index e18a4c6d60a..25283631ae5 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -72,7 +72,6 @@ def extend_build_requires(self, graph, node, build_requires_refs, check_updates, subgraph = DepsGraph() subgraph.aliased = graph.aliased - subgraph.evaluated = graph.evaluated subgraph.nodes = new_nodes for n in subgraph.nodes: n.build_require = True diff --git a/conans/client/graph/graph_manager.py b/conans/client/graph/graph_manager.py index e720847b0c7..13c59a6d7e6 100644 --- a/conans/client/graph/graph_manager.py +++ b/conans/client/graph/graph_manager.py @@ -6,7 +6,6 @@ from conans.client.graph.build_mode import BuildMode from conans.client.graph.graph import BINARY_BUILD, Node,\ RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_EDITABLE -from conans.client.graph.graph_binaries import GraphBinariesAnalyzer from conans.client.graph.graph_builder import DepsGraphBuilder from conans.errors import ConanException, conanfile_exception_formatter from conans.model.conan_file import get_env_context_manager @@ -43,13 +42,14 @@ def __str__(self): class GraphManager(object): - def __init__(self, output, cache, remote_manager, loader, proxy, resolver): + def __init__(self, output, cache, remote_manager, loader, proxy, resolver, binary_analyzer): self._proxy = proxy self._output = output self._resolver = resolver self._cache = cache self._remote_manager = remote_manager self._loader = loader + self._binary_analyzer = binary_analyzer def load_consumer_conanfile(self, conanfile_path, info_folder, deps_info_required=False, test=None): @@ -211,7 +211,7 @@ def _get_recipe_build_requires(conanfile): return conanfile.build_requires - def _recurse_build_requires(self, graph, subgraph, builder, binaries_analyzer, check_updates, + def _recurse_build_requires(self, graph, subgraph, builder, check_updates, update, build_mode, remotes, profile_build_requires, recorder, processed_profile, graph_lock, apply_build_requires=True): """ @@ -221,7 +221,7 @@ def _recurse_build_requires(self, graph, subgraph, builder, binaries_analyzer, c computed, and they will resolve build_requires if they need to be built from sources """ - binaries_analyzer.evaluate_graph(subgraph, build_mode, update, remotes) + self._binary_analyzer.evaluate_graph(subgraph, build_mode, update, remotes) if not apply_build_requires: return @@ -241,21 +241,21 @@ def _recurse_build_requires(self, graph, subgraph, builder, binaries_analyzer, c if ((node.recipe == RECIPE_CONSUMER and pattern == "&") or (node.recipe != RECIPE_CONSUMER and pattern == "&!") or fnmatch.fnmatch(str_ref, pattern)): - for build_require in build_requires: - if build_require.name in package_build_requires: # Override defined - # this is a way to have only one package Name for all versions - # (no conflicts) - # but the dict key is not used at all - package_build_requires[build_require.name] = build_require - elif build_require.name != node.name: # Profile one - new_profile_build_requires.append(build_require) + for build_require in build_requires: + if build_require.name in package_build_requires: # Override defined + # this is a way to have only one package Name for all versions + # (no conflicts) + # but the dict key is not used at all + package_build_requires[build_require.name] = build_require + elif build_require.name != node.name: # Profile one + new_profile_build_requires.append(build_require) if package_build_requires: subgraph = builder.extend_build_requires(graph, node, package_build_requires.values(), check_updates, update, remotes, processed_profile, graph_lock) - self._recurse_build_requires(graph, subgraph, builder, binaries_analyzer, + self._recurse_build_requires(graph, subgraph, builder, check_updates, update, build_mode, remotes, profile_build_requires, recorder, processed_profile, graph_lock) @@ -264,7 +264,7 @@ def _recurse_build_requires(self, graph, subgraph, builder, binaries_analyzer, c subgraph = builder.extend_build_requires(graph, node, new_profile_build_requires, check_updates, update, remotes, processed_profile, graph_lock) - self._recurse_build_requires(graph, subgraph, builder, binaries_analyzer, + self._recurse_build_requires(graph, subgraph, builder, check_updates, update, build_mode, remotes, {}, recorder, processed_profile, graph_lock) @@ -278,14 +278,10 @@ def _load_graph(self, root_node, check_updates, update, build_mode, remotes, recorder) graph = builder.load_graph(root_node, check_updates, update, remotes, processed_profile, graph_lock) - binaries_analyzer = GraphBinariesAnalyzer(self._cache, self._output, - self._remote_manager) - - self._recurse_build_requires(graph, graph, builder, binaries_analyzer, check_updates, update, - build_mode, remotes, - profile_build_requires, recorder, processed_profile, - graph_lock, - apply_build_requires=apply_build_requires) + + self._recurse_build_requires(graph, graph, builder, check_updates, update, build_mode, + remotes, profile_build_requires, recorder, processed_profile, + graph_lock, apply_build_requires=apply_build_requires) # Sort of closures, for linking order inverse_levels = {n: i for i, level in enumerate(graph.inverse_levels()) for n in level} diff --git a/conans/client/installer.py b/conans/client/installer.py index dac9c92a717..f8e28a3e0d9 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -5,8 +5,8 @@ from conans.client import tools from conans.client.file_copier import report_copied_files from conans.client.generators import TXTGenerator, write_generators -from conans.client.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_MISSING, \ - BINARY_SKIP, BINARY_UPDATE, BINARY_EDITABLE +from conans.client.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_EDITABLE, \ + BINARY_MISSING, BINARY_SKIP, BINARY_UPDATE from conans.client.importer import remove_imports, run_imports from conans.client.packager import create_package, update_package_metadata from conans.client.recorder.action_recorder import INSTALL_ERROR_BUILDING, INSTALL_ERROR_MISSING, \ @@ -19,6 +19,7 @@ from conans.model.conan_file import get_env_context_manager from conans.model.editable_layout import EditableLayout from conans.model.env_info import EnvInfo +from conans.model.graph_info import GraphInfo from conans.model.manifest import FileTreeManifest from conans.model.ref import PackageReference from conans.model.user_info import UserInfo @@ -28,7 +29,6 @@ set_dirty_context_manager) from conans.util.log import logger from conans.util.tracer import log_package_built, log_package_got_from_local_cache -from conans.model.graph_info import GraphInfo def build_id(conan_file): diff --git a/conans/test/functional/graph/graph_manager_base.py b/conans/test/functional/graph/graph_manager_base.py index 77ddec0b90f..1dafa9253c9 100644 --- a/conans/test/functional/graph/graph_manager_base.py +++ b/conans/test/functional/graph/graph_manager_base.py @@ -6,6 +6,7 @@ from conans.client.cache.cache import ClientCache from conans.client.cache.remote_registry import Remotes +from conans.client.graph.graph_binaries import GraphBinariesAnalyzer from conans.client.graph.graph_manager import GraphManager from conans.client.graph.proxy import ConanProxy from conans.client.graph.python_requires import ConanPythonRequire @@ -31,17 +32,21 @@ def setUp(self): cache_folder = temp_folder() cache = ClientCache(cache_folder, self.output) self.cache = cache + + def _get_app(self): self.remote_manager = MockRemoteManager() - self.resolver = RangeResolver(cache, self.remote_manager) + cache = self.cache + self.resolver = RangeResolver(self.cache, self.remote_manager) proxy = ConanProxy(cache, self.output, self.remote_manager) self.loader = ConanFileLoader(None, self.output, ConanPythonRequire(None, None)) + binaries = GraphBinariesAnalyzer(cache, self.output, self.remote_manager) self.manager = GraphManager(self.output, cache, self.remote_manager, self.loader, proxy, - self.resolver) + self.resolver, binaries) hook_manager = Mock() recorder = Mock() - app_type = namedtuple("ConanApp", "cache out remote_manager hook_manager") - app = app_type(self.cache, self.output, self.remote_manager, hook_manager) - self.binary_installer = BinaryInstaller(app, recorder) + app_type = namedtuple("ConanApp", "cache out remote_manager hook_manager graph_manager") + app = app_type(self.cache, self.output, self.remote_manager, hook_manager, self.manager) + return app def _cache_recipe(self, ref, test_conanfile, revision=None): if isinstance(test_conanfile, GenConanfile): @@ -60,7 +65,6 @@ def build_graph(self, content, profile_build_requires=None, ref=None, create_ref path = temp_folder() path = os.path.join(path, "conanfile.py") save(path, str(content)) - self.loader.cached_conanfiles = {} profile = Profile() if profile_build_requires: @@ -73,11 +77,13 @@ def build_graph(self, content, profile_build_requires=None, ref=None, create_ref ref = ref or ConanFileReference(None, None, None, None, validate=False) options = OptionsValues() graph_info = GraphInfo(profile, options, root_ref=ref) - deps_graph, _ = self.manager.load_graph(path, create_ref, graph_info, - build_mode, check_updates, update, - remotes, recorder) + app = self._get_app() + deps_graph, _ = app.graph_manager.load_graph(path, create_ref, graph_info, + build_mode, check_updates, update, + remotes, recorder) if install: - self.binary_installer.install(deps_graph, None, False, graph_info) + binary_installer = BinaryInstaller(app, recorder) + binary_installer.install(deps_graph, None, False, graph_info) return deps_graph def _check_node(self, node, ref, deps, build_deps, dependents, closure):