From bf97d84a3ff7f0b17840dc40fc86e1af98a153de Mon Sep 17 00:00:00 2001 From: Luis Date: Wed, 6 Jul 2022 14:01:14 +0200 Subject: [PATCH 1/7] validate_build over settings --- conans/client/graph/graph_binaries.py | 15 +++++++ .../integration/graph/test_validate_build.py | 42 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 conans/test/integration/graph/test_validate_build.py diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 103832b2a03..1b13257ee48 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -50,6 +50,7 @@ def _evaluate_build(node, build_mode): conanfile.output.info('Forced build from source') node.binary = BINARY_BUILD node.prev = None + _call_validate_build(node) return True def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, pref): @@ -178,6 +179,7 @@ def _evaluate_node(self, node, build_mode, update, remotes): pref = PackageReference(locked.ref, locked.package_id, locked.prev) # Keep locked PREV self._process_node(node, pref, build_mode, update, remotes) if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): + _call_validate_build(node) node.binary = BINARY_BUILD if node.binary == BINARY_BUILD: locked.unlock_prev() @@ -232,6 +234,7 @@ def _evaluate_node(self, node, build_mode, update, remotes): node.binary = BINARY_INVALID if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): node.binary = BINARY_BUILD + _call_validate_build(conanfile) if locked: # package_id was not locked, this means a base lockfile that is being completed @@ -295,6 +298,7 @@ def _process_node(self, node, pref, build_mode, update, remotes): conanfile.output.info("Outdated package!") node.binary = BINARY_BUILD node.prev = None + _call_validate_build(conanfile) else: conanfile.output.info("Package is up to date") @@ -451,3 +455,14 @@ def reevaluate_node(self, node, remotes, build_mode, update): output.info("Binary for updated ID from: %s" % node.binary) if node.binary == BINARY_BUILD: output.info("Binary for the updated ID has to be built") + + +def _call_validate_build(node): + conanfile = node.conanfile + if hasattr(conanfile, "validate_build") and callable(conanfile.validate_build): + with conanfile_exception_formatter(str(conanfile), "validate_build"): + try: + conanfile.validate_build() + except ConanInvalidConfiguration as e: + conanfile.info.invalid = str(e) + node.binary = BINARY_INVALID diff --git a/conans/test/integration/graph/test_validate_build.py b/conans/test/integration/graph/test_validate_build.py new file mode 100644 index 00000000000..6bce06111ef --- /dev/null +++ b/conans/test/integration/graph/test_validate_build.py @@ -0,0 +1,42 @@ +import textwrap + +from conans.test.utils.tools import TestClient + + +def test_basic_validate_build_test(): + + t = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans.errors import ConanInvalidConfiguration + + class myConan(ConanFile): + name = "foo" + version = "1.0" + settings = "os", "arch", "compiler" + + def validate_build(self): + if self.settings.compiler == "gcc": + raise ConanInvalidConfiguration("This doesn't build in GCC") + + def package_id(self): + del self.info.settings.compiler + """) + + settings_gcc = "-s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=11" + settings_clang = "-s compiler=clang -s compiler.libcxx=libc++ -s compiler.version=8" + + t.save({"conanfile.py": conanfile}) + t.run(f"create . {settings_gcc}", assert_error=True) + + assert "This doesn't build in GCC" in t.out + + t.run(f"create . {settings_clang}") + + # Now with GCC again, but now we have the binary, we don't need to build, so it doesn't fail + t.run(f"create . {settings_gcc} --build missing") + assert "foo/1.0: Already installed!" in t.out + + # But if I force the build... it will fail + t.run(f"create . {settings_gcc} ", assert_error=True) + assert "This doesn't build in GCC" in t.out From a1c998d80c29df162c23646404c04d1b88b80146 Mon Sep 17 00:00:00 2001 From: Luis Date: Thu, 7 Jul 2022 09:33:41 +0200 Subject: [PATCH 2/7] improvements --- conans/client/conan_command_output.py | 4 ++ conans/client/graph/graph.py | 2 + conans/client/graph/graph_binaries.py | 41 +++++++++---------- conans/client/graph/graph_builder.py | 10 ++++- conans/client/graph/graph_manager.py | 3 +- conans/client/installer.py | 8 +++- .../integration/graph/test_validate_build.py | 16 +++++++- 7 files changed, 58 insertions(+), 26 deletions(-) diff --git a/conans/client/conan_command_output.py b/conans/client/conan_command_output.py index 588b48240b6..91a6fff6a8d 100644 --- a/conans/client/conan_command_output.py +++ b/conans/client/conan_command_output.py @@ -140,6 +140,10 @@ def _grab_info_data(self, deps_graph, grab_paths): item_data["build_id"] = build_id(conanfile) item_data["context"] = conanfile.context + item_data["invalid_build"] = node.cant_build is not False + if node.cant_build: + item_data["invalid_build_reason"] = node.cant_build + python_requires = getattr(conanfile, "python_requires", None) if python_requires and not isinstance(python_requires, dict): # no old python requires item_data["python_requires"] = [repr(r) diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index dbfc5ea5ce2..fbcfac5c5e7 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -91,6 +91,8 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None): self.id_direct_prefs = None self.id_indirect_prefs = None + self.cant_build = False # It will set to a str with a reason if the validate_build() fails + @property def id(self): return self._id diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 1b13257ee48..6dafb7b998a 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -48,9 +48,11 @@ def _evaluate_build(node, build_mode): break if build_mode.forced(conanfile, ref, with_deps_to_build): conanfile.output.info('Forced build from source') - node.binary = BINARY_BUILD - node.prev = None - _call_validate_build(node) + if node.cant_build: + node.binary = BINARY_INVALID + else: + node.binary = BINARY_BUILD + node.prev = None return True def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, pref): @@ -179,8 +181,10 @@ def _evaluate_node(self, node, build_mode, update, remotes): pref = PackageReference(locked.ref, locked.package_id, locked.prev) # Keep locked PREV self._process_node(node, pref, build_mode, update, remotes) if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): - _call_validate_build(node) - node.binary = BINARY_BUILD + if node.cant_build: + node.binary = BINARY_INVALID + else: + node.binary = BINARY_BUILD if node.binary == BINARY_BUILD: locked.unlock_prev() @@ -233,8 +237,10 @@ def _evaluate_node(self, node, build_mode, update, remotes): if node.binary == BINARY_MISSING and node.package_id == PACKAGE_ID_INVALID: node.binary = BINARY_INVALID if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): - node.binary = BINARY_BUILD - _call_validate_build(conanfile) + if node.cant_build: + node.binary = BINARY_INVALID + else: + node.binary = BINARY_BUILD if locked: # package_id was not locked, this means a base lockfile that is being completed @@ -256,6 +262,8 @@ def _process_node(self, node, pref, build_mode, update, remotes): node.binary = BINARY_INVALID return + + if self._evaluate_build(node, build_mode): return @@ -296,9 +304,11 @@ def _process_node(self, node, pref, build_mode, update, remotes): local_recipe_hash = package_layout.recipe_manifest().summary_hash if local_recipe_hash != recipe_hash: conanfile.output.info("Outdated package!") - node.binary = BINARY_BUILD - node.prev = None - _call_validate_build(conanfile) + if node.cant_build: + node.binary = BINARY_INVALID + else: + node.binary = BINARY_BUILD + node.prev = None else: conanfile.output.info("Package is up to date") @@ -455,14 +465,3 @@ def reevaluate_node(self, node, remotes, build_mode, update): output.info("Binary for updated ID from: %s" % node.binary) if node.binary == BINARY_BUILD: output.info("Binary for the updated ID has to be built") - - -def _call_validate_build(node): - conanfile = node.conanfile - if hasattr(conanfile, "validate_build") and callable(conanfile.validate_build): - with conanfile_exception_formatter(str(conanfile), "validate_build"): - try: - conanfile.validate_build() - except ConanInvalidConfiguration as e: - conanfile.info.invalid = str(e) - node.binary = BINARY_INVALID diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 9f9d3baa69f..3e3bfee6ca7 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -3,7 +3,7 @@ from conans.client.conanfile.configure import run_configure_method from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST, CONTEXT_BUILD from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod, - conanfile_exception_formatter) + conanfile_exception_formatter, ConanInvalidConfiguration) from conans.model.conan_file import get_env_context_manager from conans.model.ref import ConanFileReference from conans.model.requires import Requirements, Requirement @@ -471,4 +471,12 @@ def _create_new_node(self, current_node, dep_graph, requirement, check_updates, dep_graph.add_node(new_node) dep_graph.add_edge(current_node, new_node, requirement) + + if hasattr(dep_conanfile, "validate_build") and callable(dep_conanfile.validate_build): + with conanfile_exception_formatter(str(dep_conanfile), "validate_build"): + try: + dep_conanfile.validate_build() + except ConanInvalidConfiguration as e: + new_node.cant_build = str(e) + return new_node diff --git a/conans/client/graph/graph_manager.py b/conans/client/graph/graph_manager.py index 3568566e77c..ca8d6871217 100644 --- a/conans/client/graph/graph_manager.py +++ b/conans/client/graph/graph_manager.py @@ -9,7 +9,7 @@ from conans.client.graph.graph_binaries import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_EDITABLE, \ BINARY_UNKNOWN from conans.client.graph.graph_builder import DepsGraphBuilder -from conans.errors import ConanException, conanfile_exception_formatter +from conans.errors import ConanException, conanfile_exception_formatter, ConanInvalidConfiguration from conans.model.conan_file import get_env_context_manager from conans.model.graph_info import GraphInfo from conans.model.graph_lock import GraphLock, GraphLockFile @@ -124,6 +124,7 @@ def load_graph(self, reference, create_reference, graph_info, build_mode, check_ root_node = self._load_root_node(reference, create_reference, profile_host, graph_lock, root_ref, lockfile_node_id, is_build_require, require_overrides) + deps_graph = self._resolve_graph(root_node, profile_host, profile_build, graph_lock, build_mode, check_updates, update, remotes, recorder, apply_build_requires=apply_build_requires) diff --git a/conans/client/installer.py b/conans/client/installer.py index 65649b942d0..260c7dcdd6b 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -423,7 +423,13 @@ def _build(self, nodes_by_level, keep_build, root_node, profile_host, profile_bu if invalid: msg = ["There are invalid packages (packages that cannot exist for this configuration):"] for node in invalid: - msg.append("{}: Invalid ID: {}".format(node.conanfile, node.conanfile.info.invalid)) + if node.cant_build: + msg.append("{}: Cannot build " + "for this configuration: {}".format(node.conanfile, + node.cant_build)) + else: + msg.append("{}: Invalid ID: {}".format(node.conanfile, + node.conanfile.info.invalid)) raise ConanInvalidConfiguration("\n".join(msg)) self._raise_missing(missing) processed_package_refs = {} diff --git a/conans/test/integration/graph/test_validate_build.py b/conans/test/integration/graph/test_validate_build.py index 6bce06111ef..e565b69af3b 100644 --- a/conans/test/integration/graph/test_validate_build.py +++ b/conans/test/integration/graph/test_validate_build.py @@ -1,3 +1,4 @@ +import json import textwrap from conans.test.utils.tools import TestClient @@ -29,7 +30,7 @@ def package_id(self): t.save({"conanfile.py": conanfile}) t.run(f"create . {settings_gcc}", assert_error=True) - assert "This doesn't build in GCC" in t.out + assert "foo/1.0: Cannot build for this configuration: This doesn't build in GCC" in t.out t.run(f"create . {settings_clang}") @@ -39,4 +40,15 @@ def package_id(self): # But if I force the build... it will fail t.run(f"create . {settings_gcc} ", assert_error=True) - assert "This doesn't build in GCC" in t.out + assert "foo/1.0: Cannot build for this configuration: This doesn't build in GCC" in t.out + + # What happens with a conan info? + t.run(f"info foo/1.0@ {settings_gcc} --json=myjson") + myjson = json.loads(t.load("myjson")) + assert myjson[0]["invalid_build"] is True + assert myjson[0]["invalid_build_reason"] == "This doesn't build in GCC" + + t.run(f"info foo/1.0@ {settings_clang} --json=myjson") + myjson = json.loads(t.load("myjson")) + assert myjson[0]["invalid_build"] is False + assert "invalid_build_reason" not in myjson[0] From 939fd744ac94292a9dea1419f09f35eed4dcd854 Mon Sep 17 00:00:00 2001 From: Luis Date: Thu, 7 Jul 2022 09:56:06 +0200 Subject: [PATCH 3/7] prev none too --- conans/client/graph/graph_binaries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 6dafb7b998a..7301341bd4e 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -52,7 +52,7 @@ def _evaluate_build(node, build_mode): node.binary = BINARY_INVALID else: node.binary = BINARY_BUILD - node.prev = None + node.prev = None return True def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, pref): @@ -308,7 +308,7 @@ def _process_node(self, node, pref, build_mode, update, remotes): node.binary = BINARY_INVALID else: node.binary = BINARY_BUILD - node.prev = None + node.prev = None else: conanfile.output.info("Package is up to date") From 55be9e8f3c88330322fbf612de4166a81efa69ad Mon Sep 17 00:00:00 2001 From: Luis Date: Mon, 11 Jul 2022 13:38:12 +0200 Subject: [PATCH 4/7] Moved validate_build verification --- conans/client/graph/graph_builder.py | 20 +++++++---- .../integration/graph/test_validate_build.py | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 3e3bfee6ca7..d2a262dea48 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -65,7 +65,20 @@ def load_graph(self, root_node, check_updates, update, remotes, profile_host, pr t1 = time.time() self._expand_node(root_node, dep_graph, Requirements(), None, None, check_updates, update, remotes, profile_host, profile_build, graph_lock) + + # Check if the nodes can be built or not + for node in dep_graph.nodes: + dep_conanfile = node.conanfile + if hasattr(dep_conanfile, "validate_build") and callable(dep_conanfile.validate_build): + with conanfile_exception_formatter(str(dep_conanfile), "validate_build"): + try: + dep_conanfile.validate_build() + except ConanInvalidConfiguration as e: + # This 'cant_build' will be ignored if we don't have to build the node. + node.cant_build = str(e) + logger.debug("GRAPH: Time to load deps %s" % (time.time() - t1)) + return dep_graph def extend_build_requires(self, graph, node, build_requires_refs, check_updates, update, @@ -472,11 +485,4 @@ def _create_new_node(self, current_node, dep_graph, requirement, check_updates, dep_graph.add_node(new_node) dep_graph.add_edge(current_node, new_node, requirement) - if hasattr(dep_conanfile, "validate_build") and callable(dep_conanfile.validate_build): - with conanfile_exception_formatter(str(dep_conanfile), "validate_build"): - try: - dep_conanfile.validate_build() - except ConanInvalidConfiguration as e: - new_node.cant_build = str(e) - return new_node diff --git a/conans/test/integration/graph/test_validate_build.py b/conans/test/integration/graph/test_validate_build.py index e565b69af3b..ec342fa5895 100644 --- a/conans/test/integration/graph/test_validate_build.py +++ b/conans/test/integration/graph/test_validate_build.py @@ -52,3 +52,39 @@ def package_id(self): myjson = json.loads(t.load("myjson")) assert myjson[0]["invalid_build"] is False assert "invalid_build_reason" not in myjson[0] + + +def test_with_options_validate_build_test(): + t = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans.errors import ConanInvalidConfiguration + + class myConan(ConanFile): + name = "foo" + version = "1.0" + options = {"my_option": [True, False]} + default_options = {"my_option": True} + + def validate_build(self): + if not self.options.my_option: + raise ConanInvalidConfiguration("This doesn't build with False option") + + """) + t.save({"conanfile.py": conanfile}) + t.run("export .") + consumer = textwrap.dedent(""" + from conan import ConanFile + from conans.errors import ConanInvalidConfiguration + + class myConan(ConanFile): + name = "consumer" + version = "1.0" + requires = "foo/1.0" + """) + t.save({"consumer.py": consumer}) + t.run("create consumer.py --build missing -o foo/*:my_option=False", assert_error=True) + assert "foo/1.0: Cannot build for this configuration: This doesn't build " \ + "with False option" in t.out + + t.run("create consumer.py --build missing -o foo/*:my_option=True") From a8d48e4bbcefe78133e39e43046a9e73f2dd6bb4 Mon Sep 17 00:00:00 2001 From: Luis Date: Mon, 11 Jul 2022 13:39:10 +0200 Subject: [PATCH 5/7] Unused import --- conans/client/graph/graph_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/graph/graph_manager.py b/conans/client/graph/graph_manager.py index ca8d6871217..10a75d463a0 100644 --- a/conans/client/graph/graph_manager.py +++ b/conans/client/graph/graph_manager.py @@ -9,7 +9,7 @@ from conans.client.graph.graph_binaries import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_EDITABLE, \ BINARY_UNKNOWN from conans.client.graph.graph_builder import DepsGraphBuilder -from conans.errors import ConanException, conanfile_exception_formatter, ConanInvalidConfiguration +from conans.errors import ConanException, conanfile_exception_formatter from conans.model.conan_file import get_env_context_manager from conans.model.graph_info import GraphInfo from conans.model.graph_lock import GraphLock, GraphLockFile From 85664621df4c76a9d1d2e3a9f24f113b77e755c9 Mon Sep 17 00:00:00 2001 From: Luis Date: Mon, 11 Jul 2022 13:40:36 +0200 Subject: [PATCH 6/7] Simplified test --- conans/test/integration/graph/test_validate_build.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/conans/test/integration/graph/test_validate_build.py b/conans/test/integration/graph/test_validate_build.py index ec342fa5895..65ab33011b0 100644 --- a/conans/test/integration/graph/test_validate_build.py +++ b/conans/test/integration/graph/test_validate_build.py @@ -1,6 +1,7 @@ import json import textwrap +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -73,15 +74,7 @@ def validate_build(self): """) t.save({"conanfile.py": conanfile}) t.run("export .") - consumer = textwrap.dedent(""" - from conan import ConanFile - from conans.errors import ConanInvalidConfiguration - - class myConan(ConanFile): - name = "consumer" - version = "1.0" - requires = "foo/1.0" - """) + consumer = GenConanfile().with_require("foo/1.0").with_name("consumer").with_version("1.0") t.save({"consumer.py": consumer}) t.run("create consumer.py --build missing -o foo/*:my_option=False", assert_error=True) assert "foo/1.0: Cannot build for this configuration: This doesn't build " \ From ba292852b89df19c969ce1bfca8717961bc7950d Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 12 Jul 2022 13:11:34 +0200 Subject: [PATCH 7/7] Moved validate_build close to validate --- conans/client/graph/graph_binaries.py | 8 ++++++++ conans/client/graph/graph_builder.py | 11 ----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 7301341bd4e..4977e2d68c2 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -414,6 +414,14 @@ def _compute_package_id(self, node, default_package_id_mode, default_python_requ except ConanInvalidConfiguration as e: conanfile.info.invalid = str(e) + if hasattr(conanfile, "validate_build") and callable(conanfile.validate_build): + with conanfile_exception_formatter(str(conanfile), "validate_build"): + try: + conanfile.validate_build() + except ConanInvalidConfiguration as e: + # This 'cant_build' will be ignored if we don't have to build the node. + node.cant_build = str(e) + info = conanfile.info node.package_id = info.package_id() diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index d2a262dea48..05b7ef1dfba 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -66,17 +66,6 @@ def load_graph(self, root_node, check_updates, update, remotes, profile_host, pr self._expand_node(root_node, dep_graph, Requirements(), None, None, check_updates, update, remotes, profile_host, profile_build, graph_lock) - # Check if the nodes can be built or not - for node in dep_graph.nodes: - dep_conanfile = node.conanfile - if hasattr(dep_conanfile, "validate_build") and callable(dep_conanfile.validate_build): - with conanfile_exception_formatter(str(dep_conanfile), "validate_build"): - try: - dep_conanfile.validate_build() - except ConanInvalidConfiguration as e: - # This 'cant_build' will be ignored if we don't have to build the node. - node.cant_build = str(e) - logger.debug("GRAPH: Time to load deps %s" % (time.time() - t1)) return dep_graph