From 8faa061f29dc27e98f49e4f5e8a6421938139f01 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 1 Mar 2021 15:36:58 +0100 Subject: [PATCH 1/2] fix lock bundle build --- conans/model/lock_bundle.py | 53 +++++++++++-------- .../graph_lock/test_lock_bundle.py | 44 ++++++++++++--- 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/conans/model/lock_bundle.py b/conans/model/lock_bundle.py index 54bbe0497a0..8c386789adc 100644 --- a/conans/model/lock_bundle.py +++ b/conans/model/lock_bundle.py @@ -14,11 +14,12 @@ class LockBundle(object): products. The format is a json with: For every reference, for each "package_id" for that reference, list the lockfiles that contain such reference:package_id and the node indexes inside that lockfile - + lock_bundle": { "app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2": { - "package_id": { - "3bcd6800847f779e0883ee91b411aad9ddd8e83c": { + "packages": [ + { + "package_id": "3bcd6800847f779e0883ee91b411aad9ddd8e83c", "lockfiles": { "app1_windows.lock": [ "1" @@ -27,9 +28,7 @@ class LockBundle(object): "prev": null, "modified": null }, - "60fbb0a22359b4888f7ecad69bcdfcd6e70e2784": { - } - }, + ], "requires": [ "pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20", "tool/0.1@#f096d7d54098b7ad7012f9435d9c33f3" @@ -38,9 +37,6 @@ class LockBundle(object): """ def __init__(self): - """ The structure is - " - """ self._nodes = {} @staticmethod @@ -64,8 +60,16 @@ def ref_convert(r): ref_str = node.ref.full_str() ref_str = ref_convert(ref_str) ref_node = result._nodes.setdefault(ref_str, {}) - pids_node = ref_node.setdefault("package_id", {}) - pid_node = pids_node.setdefault(node.package_id, {}) + packages_node = ref_node.setdefault("packages", []) + # Find existing package_id in the list of packages + # This is the equivalend of a setdefault over a dict, but on a list + for pkg in packages_node: + if pkg["package_id"] == node.package_id: + pid_node = pkg + break + else: + pid_node = {"package_id": node.package_id} + packages_node.append(pid_node) ids = pid_node.setdefault("lockfiles", {}) # TODO: Add check that this prev is always the same pid_node["prev"] = node.prev @@ -95,16 +99,21 @@ def build_order(self): opened = list(self._nodes.keys()) while opened: current_level = [] + closed = [] for o in opened: node = self._nodes[o] requires = node.get("requires", []) - if not any(n in opened for n in requires): - current_level.append(o) - - current_level.sort() - levels.append(current_level) + if not any(n in opened for n in requires): # Doesn't have an open requires + # iterate all packages to see if some has prev=null + if any(pkg["prev"] is None for pkg in node["packages"]): + current_level.append(o) + closed.append(o) + + if current_level: + current_level.sort() + levels.append(current_level) # now initialize new level - opened = set(opened).difference(current_level) + opened = set(opened).difference(closed) return levels @@ -118,12 +127,12 @@ def update_bundle(bundle_path, revisions_enabled): bundle.loads(load(bundle_path)) for node in bundle._nodes.values(): # Each node in bundle belongs to a "ref", and contains lockinfo for every package_id - for bundle_package_ids in node["package_id"].values(): + for pkg in node["packages"]: # Each package_id contains information of multiple lockfiles # First, compute the modified PREV from all lockfiles prev = modified = prev_lockfile = None - for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items(): + for lockfile, nodes_ids in pkg["lockfiles"].items(): graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock @@ -139,11 +148,11 @@ def update_bundle(bundle_path, revisions_enabled): msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format( ref, prev_lockfile, prev, lockfile, lock_prev) raise ConanException(msg) - bundle_package_ids["prev"] = prev - bundle_package_ids["modified"] = modified + pkg["prev"] = prev + pkg["modified"] = modified # Then, update all prev of all config lockfiles - for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items(): + for lockfile, nodes_ids in pkg["lockfiles"].items(): graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock for node_id in nodes_ids: diff --git a/conans/test/integration/graph_lock/test_lock_bundle.py b/conans/test/integration/graph_lock/test_lock_bundle.py index df40d2e63c3..8f524e4b390 100644 --- a/conans/test/integration/graph_lock/test_lock_bundle.py +++ b/conans/test/integration/graph_lock/test_lock_bundle.py @@ -58,9 +58,9 @@ def test_basic(): for level in order: for ref in level: # Now get the package_id, lockfile - pkg_ids = bundle[ref]["package_id"] - for pkg_id, lockfile_info in pkg_ids.items(): - lockfiles = lockfile_info["lockfiles"] + packages = bundle[ref]["packages"] + for pkg in packages: + lockfiles = pkg["lockfiles"] lockfile = next(iter(sorted(lockfiles))) client.run("install {ref} --build={ref} --lockfile={lockfile} " @@ -166,9 +166,9 @@ def test_build_requires(): for level in order: for ref in level: # Now get the package_id, lockfile - pkg_ids = bundle[ref]["package_id"] - for pkg_id, lockfile_info in pkg_ids.items(): - lockfiles = lockfile_info["lockfiles"] + packages = bundle[ref]["packages"] + for pkg in packages: + lockfiles = pkg["lockfiles"] lockfile = next(iter(sorted(lockfiles))) client.run("install {ref} --build={ref} --lockfile={lockfile} " @@ -222,3 +222,35 @@ def test_build_requires(): assert nodes[n].ref.full_str() == "tool/0.1#f096d7d54098b7ad7012f9435d9c33f3" assert nodes[n].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31" assert nodes[n].prev == "9e99cfd92d0d7df79d687b01512ce844" + + +def test_build_requires_error(): + # https://github.com/conan-io/conan/issues/8577 + client = TestClient() + # TODO: This is hardcoded + client.run("config set general.revisions_enabled=1") + client.save({"tool/conanfile.py": GenConanfile().with_settings("os"), + "pkga/conanfile.py": GenConanfile().with_settings("os"), + "app1/conanfile.py": GenConanfile().with_settings("os").with_requires("pkga/0.1"), + "profile": "[build_requires]\ntool/0.1"}) + client.run("create tool tool/0.1@ -s os=Windows") + client.run("create tool tool/0.1@ -s os=Linux") + client.run("export pkga pkga/0.1@") + client.run("export app1 app1/0.1@") + + client.run("lock create --ref=app1/0.1 -pr=profile -s os=Windows " + "--lockfile-out=app1_windows.lock --build=missing") + assert "tool/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Cache" in client.out + client.run("lock create --ref=app1/0.1 -pr=profile -s os=Linux " + "--lockfile-out=app1_linux.lock --build=missing") + assert "tool/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Cache" in client.out + + client.run("lock bundle create app1_windows.lock app1_linux.lock --bundle-out=lock1.bundle") + client.run("lock bundle build-order lock1.bundle --json=bo.json") + order = client.load("bo.json") + print(order) + order = json.loads(order) + assert order == [ + ["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"], + ["app1/0.1@#5af607abc205b47375f485a98abc3b38"] + ] From 1de4c438481c6b9ba8db386c3b84e5a057bce10f Mon Sep 17 00:00:00 2001 From: James Date: Mon, 1 Mar 2021 17:38:26 +0100 Subject: [PATCH 2/2] Update conans/model/lock_bundle.py Co-authored-by: Jerry Wiltse --- conans/model/lock_bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/model/lock_bundle.py b/conans/model/lock_bundle.py index 8c386789adc..fa0815e4d15 100644 --- a/conans/model/lock_bundle.py +++ b/conans/model/lock_bundle.py @@ -62,7 +62,7 @@ def ref_convert(r): ref_node = result._nodes.setdefault(ref_str, {}) packages_node = ref_node.setdefault("packages", []) # Find existing package_id in the list of packages - # This is the equivalend of a setdefault over a dict, but on a list + # This is the equivalent of a setdefault over a dict, but on a list for pkg in packages_node: if pkg["package_id"] == node.package_id: pid_node = pkg