Skip to content

Commit

Permalink
fix lock bundle build (#8579)
Browse files Browse the repository at this point in the history 
* fix lock bundle build

* Update conans/model/lock_bundle.py

Co-authored-by: Jerry Wiltse <jerrywiltse@gmail.com>

Co-authored-by: Jerry Wiltse <jerrywiltse@gmail.com>
  • Loading branch information
@memsharded @solvingj
memsharded and solvingj committed Mar 1, 2021
1 parent 4bbaf8c commit 40aa789
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 28 deletions.
53 changes: 31 additions & 22 deletions conans/model/lock_bundle.py
View file
Expand Up @@ -14,11 +14,12 @@ class LockBundle(object):
products. The format is a json with: For every reference, for each "package_id" for that
reference, list the lockfiles that contain such reference:package_id and the node indexes
inside that lockfile
lock_bundle": {
"app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2": {
"package_id": {
"3bcd6800847f779e0883ee91b411aad9ddd8e83c": {
"packages": [
{
"package_id": "3bcd6800847f779e0883ee91b411aad9ddd8e83c",
"lockfiles": {
"app1_windows.lock": [
"1"
Expand All @@ -27,9 +28,7 @@ class LockBundle(object):
"prev": null,
"modified": null
},
"60fbb0a22359b4888f7ecad69bcdfcd6e70e2784": {
}
},
],
"requires": [
"pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20",
"tool/0.1@#f096d7d54098b7ad7012f9435d9c33f3"
Expand All @@ -38,9 +37,6 @@ class LockBundle(object):
"""

def __init__(self):
""" The structure is
"
"""
self._nodes = {}

@staticmethod
Expand All @@ -64,8 +60,16 @@ def ref_convert(r):
ref_str = node.ref.full_str()
ref_str = ref_convert(ref_str)
ref_node = result._nodes.setdefault(ref_str, {})
pids_node = ref_node.setdefault("package_id", {})
pid_node = pids_node.setdefault(node.package_id, {})
packages_node = ref_node.setdefault("packages", [])
# Find existing package_id in the list of packages
# This is the equivalent of a setdefault over a dict, but on a list
for pkg in packages_node:
if pkg["package_id"] == node.package_id:
pid_node = pkg
break
else:
pid_node = {"package_id": node.package_id}
packages_node.append(pid_node)
ids = pid_node.setdefault("lockfiles", {})
# TODO: Add check that this prev is always the same
pid_node["prev"] = node.prev
Expand Down Expand Up @@ -95,16 +99,21 @@ def build_order(self):
opened = list(self._nodes.keys())
while opened:
current_level = []
closed = []
for o in opened:
node = self._nodes[o]
requires = node.get("requires", [])
if not any(n in opened for n in requires):
current_level.append(o)

current_level.sort()
levels.append(current_level)
if not any(n in opened for n in requires): # Doesn't have an open requires
# iterate all packages to see if some has prev=null
if any(pkg["prev"] is None for pkg in node["packages"]):
current_level.append(o)
closed.append(o)

if current_level:
current_level.sort()
levels.append(current_level)
# now initialize new level
opened = set(opened).difference(current_level)
opened = set(opened).difference(closed)

return levels

Expand All @@ -118,12 +127,12 @@ def update_bundle(bundle_path, revisions_enabled):
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
# Each node in bundle belongs to a "ref", and contains lockinfo for every package_id
for bundle_package_ids in node["package_id"].values():
for pkg in node["packages"]:
# Each package_id contains information of multiple lockfiles

# First, compute the modified PREV from all lockfiles
prev = modified = prev_lockfile = None
for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items():
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock

Expand All @@ -139,11 +148,11 @@ def update_bundle(bundle_path, revisions_enabled):
msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format(
ref, prev_lockfile, prev, lockfile, lock_prev)
raise ConanException(msg)
bundle_package_ids["prev"] = prev
bundle_package_ids["modified"] = modified
pkg["prev"] = prev
pkg["modified"] = modified

# Then, update all prev of all config lockfiles
for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items():
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
Expand Down
44 changes: 38 additions & 6 deletions conans/test/integration/graph_lock/test_lock_bundle.py
View file
Expand Up @@ -58,9 +58,9 @@ def test_basic():
for level in order:
for ref in level:
# Now get the package_id, lockfile
pkg_ids = bundle[ref]["package_id"]
for pkg_id, lockfile_info in pkg_ids.items():
lockfiles = lockfile_info["lockfiles"]
packages = bundle[ref]["packages"]
for pkg in packages:
lockfiles = pkg["lockfiles"]
lockfile = next(iter(sorted(lockfiles)))

client.run("install {ref} --build={ref} --lockfile={lockfile} "
Expand Down Expand Up @@ -166,9 +166,9 @@ def test_build_requires():
for level in order:
for ref in level:
# Now get the package_id, lockfile
pkg_ids = bundle[ref]["package_id"]
for pkg_id, lockfile_info in pkg_ids.items():
lockfiles = lockfile_info["lockfiles"]
packages = bundle[ref]["packages"]
for pkg in packages:
lockfiles = pkg["lockfiles"]
lockfile = next(iter(sorted(lockfiles)))

client.run("install {ref} --build={ref} --lockfile={lockfile} "
Expand Down Expand Up @@ -222,3 +222,35 @@ def test_build_requires():
assert nodes[n].ref.full_str() == "tool/0.1#f096d7d54098b7ad7012f9435d9c33f3"
assert nodes[n].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31"
assert nodes[n].prev == "9e99cfd92d0d7df79d687b01512ce844"


def test_build_requires_error():
# https://github.com/conan-io/conan/issues/8577
client = TestClient()
# TODO: This is hardcoded
client.run("config set general.revisions_enabled=1")
client.save({"tool/conanfile.py": GenConanfile().with_settings("os"),
"pkga/conanfile.py": GenConanfile().with_settings("os"),
"app1/conanfile.py": GenConanfile().with_settings("os").with_requires("pkga/0.1"),
"profile": "[build_requires]\ntool/0.1"})
client.run("create tool tool/0.1@ -s os=Windows")
client.run("create tool tool/0.1@ -s os=Linux")
client.run("export pkga pkga/0.1@")
client.run("export app1 app1/0.1@")

client.run("lock create --ref=app1/0.1 -pr=profile -s os=Windows "
"--lockfile-out=app1_windows.lock --build=missing")
assert "tool/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Cache" in client.out
client.run("lock create --ref=app1/0.1 -pr=profile -s os=Linux "
"--lockfile-out=app1_linux.lock --build=missing")
assert "tool/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Cache" in client.out

client.run("lock bundle create app1_windows.lock app1_linux.lock --bundle-out=lock1.bundle")
client.run("lock bundle build-order lock1.bundle --json=bo.json")
order = client.load("bo.json")
print(order)
order = json.loads(order)
assert order == [
["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"],
["app1/0.1@#5af607abc205b47375f485a98abc3b38"]
]

0 comments on commit 40aa789

Please sign in to comment.