refactor GraphBinariesAnalyzer (#5836)

* refactor GraphBinariesAnalyzer

* 1.19.0 release

* 1.20.0-dev

* Issue/5814 fix python_requires with short_paths enabled (#5841)

* Added required = True to subparsers in order to print error message in Py2 and Py3.

* sync

* basic concurrent upload at reference level with futures

* revert changes

* add line

* Lock buggy urllib3 (#5808)

* app simplifying (#5806)

* Apply lockfile before updating downstream requires (#5771)

* apply graph_lock before looking for overrides

* first step: get rid of the warning

* cleaner if graph_lock is passed to the function

* only update requires upstream if no lockfile is applied

* fix tests

* Deprecation of CONAN_USERNAME and CONAN_CHANNEL: fix error message (#5756)

* if CONAN_USERNAME and CONAN_CHANNEL are deprecated, the error cannot recommend them

* update tests accordingly

* test client load() file method (#5815)

* no user/channel repr without _ (#5817)

* no user/channel repr without _

* minor fixes

* fix tests

* Remove py34 (#5820)

* fix upload package id (#5824)

* - update macOS, watchOS, tvOS, iOS version numbers (#5823)

* Refresh token client support.  (#5662)

* Refresh token client support. Missing tests. Missing migration

* public method

* WIP

* Refresh almost there

* Removed prints

* Try migrate

* Migration

* Add comment

* Refresh token flow following RFC recommentations

* Refresh ok

* review

* Remove traces

* Refactor capabilities

* Removed tmp file

* Review

* #5819 Show warning message for Python 3.4 (#5829)

* #5819 Show warning message for Python 3.4

- Add new warning message for python 3.4 which is no longer supported
- Added funcional tests to validate both python 3.4 and 2.x

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #5819 Fix broken tests

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Add cpp_info.name to cmake and pkg_config generators (#5598)

* Add cpp_info.name to cmake generators

* Fix unit tests to mimic real behavior

* cmake_paths test

* add test for cmake generator

* Add cmake_find_package test

* fix test in py3

* Applied cpp_info.name to pkg_config generator

* check different name in pkg_config

* use pyreq short path if exists

* sync with develop

* revert change

* pass correct conanfile

* add test

* remove line

* clean test

* Feature/remote enable error handling (#5835)

* Add a test

* Raise an error if no remote found

* Check that we don't raise an error if the state is already in the required state

* Raise an error only if the remote name is not a wildcard

* Check that we don't raise an error with a wildcard if there are no remotes

* Removed go test files and references to golang from README (#5854)

* Removed go test files and references to golang from README

* Removed golang from README

* making GraphBinaryAnalyzer injectable

conans/client/conan_api.py

@@ -2,6 +2,7 @@
 import sys
 from collections import OrderedDict
 
+from conans.client.graph.graph_binaries import GraphBinariesAnalyzer
 from conans.client.manager import deps_install
 from conans.paths.package_layouts.package_cache_layout import PackageCacheLayout
 
@@ -181,9 +182,9 @@ def __init__(self, cache_folder, user_io, http_requester=None, runner=None):
         self.python_requires = ConanPythonRequire(self.proxy, resolver)
         self.loader = ConanFileLoader(self.runner, self.out, self.python_requires)
 
-        self.graph_manager = GraphManager(self.out, self.cache,
-                                          self.remote_manager, self.loader, self.proxy,
-                                          resolver)
+        self.binaries_analyzer = GraphBinariesAnalyzer(self.cache, self.out, self.remote_manager)
+        self.graph_manager = GraphManager(self.out, self.cache, self.remote_manager, self.loader,
+                                          self.proxy, resolver, self.binaries_analyzer)
 
     def load_remotes(self, remote_name=None, update=False, check_updates=False):
         remotes = self.cache.registry.load_remotes()

conans/client/graph/graph.py

@@ -201,8 +201,6 @@ def __init__(self):
         self.nodes = set()
         self.root = None
         self.aliased = {}
-        # These are the nodes with pref (not including PREV) that have been evaluated
-        self.evaluated = {}  # {pref: [nodes]}
 
     def add_node(self, node):
         if not self.nodes:

conans/client/graph/graph_binaries.py

@@ -18,8 +18,11 @@ def __init__(self, cache, output, remote_manager):
         self._cache = cache
         self._out = output
         self._remote_manager = remote_manager
+        # These are the nodes with pref (not including PREV) that have been evaluated
+        self._evaluated = {}  # {pref: [nodes]}
 
-    def _check_update(self, upstream_manifest, package_folder, output, node):
+    @staticmethod
+    def _check_update(upstream_manifest, package_folder, output, node):
         read_manifest = FileTreeManifest.load(package_folder)
         if upstream_manifest != read_manifest:
             if upstream_manifest.time > read_manifest.time:
@@ -29,7 +32,126 @@ def _check_update(self, upstream_manifest, package_folder, output, node):
             else:
                 output.warn("Current package is newer than remote upstream one")
 
-    def _evaluate_node(self, node, build_mode, update, evaluated_nodes, remotes):
+    @staticmethod
+    def _evaluate_build(node, build_mode):
+        ref, conanfile = node.ref, node.conanfile
+        with_deps_to_build = False
+        # For cascade mode, we need to check also the "modified" status of the lockfile if exists
+        # modified nodes have already been built, so they shouldn't be built again
+        if build_mode.cascade and not (node.graph_lock_node and node.graph_lock_node.modified):
+            for dep in node.dependencies:
+                dep_node = dep.dst
+                if (dep_node.binary == BINARY_BUILD or
+                        (dep_node.graph_lock_node and dep_node.graph_lock_node.modified)):
+                    with_deps_to_build = True
+                    break
+        if build_mode.forced(conanfile, ref, with_deps_to_build):
+            conanfile.output.info('Forced build from source')
+            node.binary = BINARY_BUILD
+            node.prev = None
+            return True
+
+    def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, package_folder, pref):
+        # Check if dirty, to remove it
+        with package_layout.package_lock(pref):
+            assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code"
+            if is_dirty(package_folder):
+                node.conanfile.output.warn("Package is corrupted, removing folder: %s"
+                                           % package_folder)
+                rmdir(package_folder)  # Do not remove if it is EDITABLE
+                return
+
+            if self._cache.config.revisions_enabled:
+                metadata = package_layout.load_metadata()
+                rec_rev = metadata.packages[pref.id].recipe_revision
+                if rec_rev and rec_rev != node.ref.revision:
+                    node.conanfile.output.warn("The package {} doesn't belong to the installed "
+                                               "recipe revision, removing folder".format(pref))
+                    rmdir(package_folder)
+                return metadata
+
+    def _evaluate_cache_pkg(self, node, package_layout, pref, metadata, remote, remotes, update,
+                            package_folder):
+        if update:
+            output = node.conanfile.output
+            if remote:
+                try:
+                    tmp = self._remote_manager.get_package_manifest(pref, remote)
+                    upstream_manifest, pref = tmp
+                except NotFoundException:
+                    output.warn("Can't update, no package in remote")
+                except NoRemoteAvailable:
+                    output.warn("Can't update, no remote defined")
+                else:
+                    if self._check_update(upstream_manifest, package_folder, output, node):
+                        node.binary = BINARY_UPDATE
+                        node.prev = pref.revision  # With revision
+            elif remotes:
+                pass  # Current behavior: no remote explicit or in metadata, do not update
+            else:
+                output.warn("Can't update, no remote defined")
+        if not node.binary:
+            node.binary = BINARY_CACHE
+            metadata = metadata or package_layout.load_metadata()
+            node.prev = metadata.packages[pref.id].revision
+            assert node.prev, "PREV for %s is None: %s" % (str(pref), metadata.dumps())
+
+    def _evaluate_remote_pkg(self, node, pref, remote, remotes, build_mode):
+        remote_info = None
+        if remote:
+            try:
+                remote_info, pref = self._remote_manager.get_package_info(pref, remote)
+            except NotFoundException:
+                pass
+            except Exception:
+                node.conanfile.output.error("Error downloading binary package: '{}'".format(pref))
+                raise
+
+        # If the "remote" came from the registry but the user didn't specified the -r, with
+        # revisions iterate all remotes
+        if not remote or (not remote_info and self._cache.config.revisions_enabled):
+            for r in remotes.values():
+                try:
+                    remote_info, pref = self._remote_manager.get_package_info(pref, r)
+                except NotFoundException:
+                    pass
+                else:
+                    if remote_info:
+                        remote = r
+                        break
+
+        if remote_info:
+            node.binary = BINARY_DOWNLOAD
+            node.prev = pref.revision
+            recipe_hash = remote_info.recipe_hash
+        else:
+            recipe_hash = None
+            if build_mode.allowed(node.conanfile):
+                node.binary = BINARY_BUILD
+            else:
+                node.binary = BINARY_MISSING
+            node.prev = None
+
+        return recipe_hash, remote
+
+    def _evaluate_is_cached(self, node, pref):
+        previous_nodes = self._evaluated.get(pref)
+        if previous_nodes:
+            previous_nodes.append(node)
+            previous_node = previous_nodes[0]
+            # The previous node might have been skipped, but current one not necessarily
+            # keep the original node.binary value (before being skipped), and if it will be
+            # defined as SKIP again by self._handle_private(node) if it is really private
+            if previous_node.binary == BINARY_SKIP:
+                node.binary = previous_node.binary_non_skip
+            else:
+                node.binary = previous_node.binary
+            node.binary_remote = previous_node.binary_remote
+            node.prev = previous_node.prev
+            return True
+        self._evaluated[pref] = [node]
+
+    def _evaluate_node(self, node, build_mode, update, remotes):
         assert node.binary is None, "Node.binary should be None"
         assert node.package_id is not None, "Node.package_id shouldn't be None"
         assert node.prev is None, "Node.prev should be None"
@@ -48,145 +170,51 @@ def _evaluate_node(self, node, build_mode, update, evaluated_nodes, remotes):
             pref = PackageReference(ref, node.package_id)
 
         # Check that this same reference hasn't already been checked
-        previous_nodes = evaluated_nodes.get(pref)
-        if previous_nodes:
-            previous_nodes.append(node)
-            previous_node = previous_nodes[0]
-            # The previous node might have been skipped, but current one not necessarily
-            # keep the original node.binary value (before being skipped), and if it will be
-            # defined as SKIP again by self._handle_private(node) if it is really private
-            if previous_node.binary == BINARY_SKIP:
-                node.binary = previous_node.binary_non_skip
-            else:
-                node.binary = previous_node.binary
-            node.binary_remote = previous_node.binary_remote
-            node.prev = previous_node.prev
+        if self._evaluate_is_cached(node, pref):
             return
-        evaluated_nodes[pref] = [node]
-
-        output = conanfile.output
 
         if node.recipe == RECIPE_EDITABLE:
-            node.binary = BINARY_EDITABLE
-            # TODO: PREV?
+            node.binary = BINARY_EDITABLE  # TODO: PREV?
             return
 
-        with_deps_to_build = False
-        # For cascade mode, we need to check also the "modified" status of the lockfile if exists
-        # modified nodes have already been built, so they shouldn't be built again
-        if build_mode.cascade and not (node.graph_lock_node and node.graph_lock_node.modified):
-            for dep in node.dependencies:
-                dep_node = dep.dst
-                if (dep_node.binary == BINARY_BUILD or
-                        (dep_node.graph_lock_node and dep_node.graph_lock_node.modified)):
-                    with_deps_to_build = True
-                    break
-        if build_mode.forced(conanfile, ref, with_deps_to_build):
-            output.info('Forced build from source')
-            node.binary = BINARY_BUILD
-            node.prev = None
+        if self._evaluate_build(node, build_mode):
             return
 
-        package_folder = self._cache.package_layout(pref.ref,
-                                                    short_paths=conanfile.short_paths).package(pref)
-
-        # Check if dirty, to remove it
-        with self._cache.package_layout(pref.ref).package_lock(pref):
-            assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code"
-            if is_dirty(package_folder):
-                output.warn("Package is corrupted, removing folder: %s" % package_folder)
-                rmdir(package_folder)  # Do not remove if it is EDITABLE
-
-            if self._cache.config.revisions_enabled:
-                metadata = self._cache.package_layout(pref.ref).load_metadata()
-                rec_rev = metadata.packages[pref.id].recipe_revision
-                if rec_rev and rec_rev != node.ref.revision:
-                    output.warn("The package {} doesn't belong "
-                                "to the installed recipe revision, removing folder".format(pref))
-                    rmdir(package_folder)
+        package_layout = self._cache.package_layout(pref.ref, short_paths=conanfile.short_paths)
+        package_folder = package_layout.package(pref)
+        metadata = self._evaluate_clean_pkg_folder_dirty(node, package_layout, package_folder, pref)
 
         remote = remotes.selected
         if not remote:
-            # If the remote_name is not given, follow the binary remote, or
-            # the recipe remote
+            # If the remote_name is not given, follow the binary remote, or the recipe remote
             # If it is defined it won't iterate (might change in conan2.0)
-            metadata = self._cache.package_layout(pref.ref).load_metadata()
+            metadata = metadata or package_layout.load_metadata()
             remote_name = metadata.packages[pref.id].remote or metadata.recipe.remote
             remote = remotes.get(remote_name)
 
-        if os.path.exists(package_folder):
-            if update:
-                if remote:
-                    try:
-                        tmp = self._remote_manager.get_package_manifest(pref, remote)
-                        upstream_manifest, pref = tmp
-                    except NotFoundException:
-                        output.warn("Can't update, no package in remote")
-                    except NoRemoteAvailable:
-                        output.warn("Can't update, no remote defined")
-                    else:
-                        if self._check_update(upstream_manifest, package_folder, output, node):
-                            node.binary = BINARY_UPDATE
-                            node.prev = pref.revision  # With revision
-                            if build_mode.outdated:
-                                info, pref = self._remote_manager.get_package_info(pref, remote)
-                                package_hash = info.recipe_hash
-                elif remotes:
-                    pass
-                else:
-                    output.warn("Can't update, no remote defined")
-            if not node.binary:
-                node.binary = BINARY_CACHE
-                metadata = self._cache.package_layout(pref.ref).load_metadata()
-                node.prev = metadata.packages[pref.id].revision
-                assert node.prev, "PREV for %s is None: %s" % (str(pref), metadata.dumps())
-                package_hash = ConanInfo.load_from_package(package_folder).recipe_hash
-
+        if os.path.exists(package_folder):  # Binary already in cache, check for updates
+            self._evaluate_cache_pkg(node, package_layout, pref, metadata,  remote, remotes, update,
+                                     package_folder)
+            recipe_hash = None
         else:  # Binary does NOT exist locally
-            remote_info = None
-            if remote:
-                try:
-                    remote_info, pref = self._remote_manager.get_package_info(pref, remote)
-                except NotFoundException:
-                    pass
-                except Exception:
-                    conanfile.output.error("Error downloading binary package: '{}'".format(pref))
-                    raise
-
-            # If the "remote" came from the registry but the user didn't specified the -r, with
-            # revisions iterate all remotes
-
-            if not remote or (not remote_info and self._cache.config.revisions_enabled):
-                for r in remotes.values():
-                    try:
-                        remote_info, pref = self._remote_manager.get_package_info(pref, r)
-                    except NotFoundException:
-                        pass
-                    else:
-                        if remote_info:
-                            remote = r
-                            break
-
-            if remote_info:
-                node.binary = BINARY_DOWNLOAD
-                node.prev = pref.revision
-                package_hash = remote_info.recipe_hash
-            else:
-                if build_mode.allowed(conanfile):
-                    node.binary = BINARY_BUILD
-                else:
-                    node.binary = BINARY_MISSING
-                node.prev = None
+            # Returned remote might be different than the passed one if iterating remotes
+            recipe_hash, remote = self._evaluate_remote_pkg(node, pref, remote, remotes, build_mode)
 
         if build_mode.outdated:
             if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE):
-                local_recipe_hash = self._cache.package_layout(ref).recipe_manifest().summary_hash
-                if local_recipe_hash != package_hash:
-                    output.info("Outdated package!")
+                if node.binary == BINARY_UPDATE:
+                    info, pref = self._remote_manager.get_package_info(pref, remote)
+                    recipe_hash = info.recipe_hash
+                elif node.binary == BINARY_CACHE:
+                    recipe_hash = ConanInfo.load_from_package(package_folder).recipe_hash
+
+                local_recipe_hash = package_layout.recipe_manifest().summary_hash
+                if local_recipe_hash != recipe_hash:
+                    conanfile.output.info("Outdated package!")
                     node.binary = BINARY_BUILD
                     node.prev = None
                 else:
-                    output.info("Package is up to date")
+                    conanfile.output.info("Package is up to date")
 
         node.binary_remote = remote
 
@@ -241,10 +269,9 @@ def _handle_private(self, node):
 
     def evaluate_graph(self, deps_graph, build_mode, update, remotes):
         default_package_id_mode = self._cache.config.default_package_id_mode
-        evaluated = deps_graph.evaluated
         for node in deps_graph.ordered_iterate():
             self._compute_package_id(node, default_package_id_mode)
             if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
                 continue
-            self._evaluate_node(node, build_mode, update, evaluated, remotes)
+            self._evaluate_node(node, build_mode, update, remotes)
             self._handle_private(node)

conans/client/graph/graph_builder.py

@@ -72,7 +72,6 @@ def extend_build_requires(self, graph, node, build_requires_refs, check_updates,
 
         subgraph = DepsGraph()
         subgraph.aliased = graph.aliased
-        subgraph.evaluated = graph.evaluated
         subgraph.nodes = new_nodes
         for n in subgraph.nodes:
             n.build_require = True