Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix lock bundle build #8579

Merged
merged 2 commits into from Mar 1, 2021
Merged

fix lock bundle build #8579

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8faa061
fix lock bundle build
memsharded Mar 1, 2021
1de4c43
Update conans/model/lock_bundle.py
memsharded Mar 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
53 changes: 31 additions & 22 deletions conans/model/lock_bundle.py
View file
Open in desktop
Expand Up @@ -14,11 +14,12 @@ class LockBundle(object):
products. The format is a json with: For every reference, for each "package_id" for that
reference, list the lockfiles that contain such reference:package_id and the node indexes
inside that lockfile

lock_bundle": {
"app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2": {
"package_id": {
"3bcd6800847f779e0883ee91b411aad9ddd8e83c": {
"packages": [
{
"package_id": "3bcd6800847f779e0883ee91b411aad9ddd8e83c",
"lockfiles": {
"app1_windows.lock": [
"1"
Expand All @@ -27,9 +28,7 @@ class LockBundle(object):
"prev": null,
"modified": null
},
"60fbb0a22359b4888f7ecad69bcdfcd6e70e2784": {
}
},
],
"requires": [
"pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20",
"tool/0.1@#f096d7d54098b7ad7012f9435d9c33f3"
Expand All @@ -38,9 +37,6 @@ class LockBundle(object):
"""

def __init__(self):
""" The structure is
"
"""
self._nodes = {}

@staticmethod
Expand All @@ -64,8 +60,16 @@ def ref_convert(r):
ref_str = node.ref.full_str()
ref_str = ref_convert(ref_str)
ref_node = result._nodes.setdefault(ref_str, {})
pids_node = ref_node.setdefault("package_id", {})
pid_node = pids_node.setdefault(node.package_id, {})
packages_node = ref_node.setdefault("packages", [])
# Find existing package_id in the list of packages
# This is the equivalend of a setdefault over a dict, but on a list
memsharded marked this conversation as resolved.
Show resolved Hide resolved
for pkg in packages_node:
if pkg["package_id"] == node.package_id:
pid_node = pkg
break
else:
pid_node = {"package_id": node.package_id}
packages_node.append(pid_node)
ids = pid_node.setdefault("lockfiles", {})
# TODO: Add check that this prev is always the same
pid_node["prev"] = node.prev
Expand Down Expand Up @@ -95,16 +99,21 @@ def build_order(self):
opened = list(self._nodes.keys())
while opened:
current_level = []
closed = []
for o in opened:
node = self._nodes[o]
requires = node.get("requires", [])
if not any(n in opened for n in requires):
current_level.append(o)

current_level.sort()
levels.append(current_level)
if not any(n in opened for n in requires): # Doesn't have an open requires
# iterate all packages to see if some has prev=null
if any(pkg["prev"] is None for pkg in node["packages"]):
current_level.append(o)
closed.append(o)

if current_level:
current_level.sort()
levels.append(current_level)
# now initialize new level
opened = set(opened).difference(current_level)
opened = set(opened).difference(closed)

return levels

Expand All @@ -118,12 +127,12 @@ def update_bundle(bundle_path, revisions_enabled):
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
# Each node in bundle belongs to a "ref", and contains lockinfo for every package_id
for bundle_package_ids in node["package_id"].values():
for pkg in node["packages"]:
# Each package_id contains information of multiple lockfiles

# First, compute the modified PREV from all lockfiles
prev = modified = prev_lockfile = None
for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items():
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock

Expand All @@ -139,11 +148,11 @@ def update_bundle(bundle_path, revisions_enabled):
msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format(
ref, prev_lockfile, prev, lockfile, lock_prev)
raise ConanException(msg)
bundle_package_ids["prev"] = prev
bundle_package_ids["modified"] = modified
pkg["prev"] = prev
pkg["modified"] = modified

# Then, update all prev of all config lockfiles
for lockfile, nodes_ids in bundle_package_ids["lockfiles"].items():
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
Expand Down
44 changes: 38 additions & 6 deletions conans/test/integration/graph_lock/test_lock_bundle.py
View file
Open in desktop
Expand Up @@ -58,9 +58,9 @@ def test_basic():
for level in order:
for ref in level:
# Now get the package_id, lockfile
pkg_ids = bundle[ref]["package_id"]
for pkg_id, lockfile_info in pkg_ids.items():
lockfiles = lockfile_info["lockfiles"]
packages = bundle[ref]["packages"]
for pkg in packages:
lockfiles = pkg["lockfiles"]
lockfile = next(iter(sorted(lockfiles)))

client.run("install {ref} --build={ref} --lockfile={lockfile} "
Expand Down Expand Up @@ -166,9 +166,9 @@ def test_build_requires():
for level in order:
for ref in level:
# Now get the package_id, lockfile
pkg_ids = bundle[ref]["package_id"]
for pkg_id, lockfile_info in pkg_ids.items():
lockfiles = lockfile_info["lockfiles"]
packages = bundle[ref]["packages"]
for pkg in packages:
lockfiles = pkg["lockfiles"]
lockfile = next(iter(sorted(lockfiles)))

client.run("install {ref} --build={ref} --lockfile={lockfile} "
Expand Down Expand Up @@ -222,3 +222,35 @@ def test_build_requires():
assert nodes[n].ref.full_str() == "tool/0.1#f096d7d54098b7ad7012f9435d9c33f3"
assert nodes[n].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31"
assert nodes[n].prev == "9e99cfd92d0d7df79d687b01512ce844"


def test_build_requires_error():
# https://github.com/conan-io/conan/issues/8577
client = TestClient()
# TODO: This is hardcoded
client.run("config set general.revisions_enabled=1")
client.save({"tool/conanfile.py": GenConanfile().with_settings("os"),
"pkga/conanfile.py": GenConanfile().with_settings("os"),
"app1/conanfile.py": GenConanfile().with_settings("os").with_requires("pkga/0.1"),
"profile": "[build_requires]\ntool/0.1"})
client.run("create tool tool/0.1@ -s os=Windows")
client.run("create tool tool/0.1@ -s os=Linux")
client.run("export pkga pkga/0.1@")
client.run("export app1 app1/0.1@")

client.run("lock create --ref=app1/0.1 -pr=profile -s os=Windows "
"--lockfile-out=app1_windows.lock --build=missing")
assert "tool/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Cache" in client.out
client.run("lock create --ref=app1/0.1 -pr=profile -s os=Linux "
"--lockfile-out=app1_linux.lock --build=missing")
assert "tool/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Cache" in client.out

client.run("lock bundle create app1_windows.lock app1_linux.lock --bundle-out=lock1.bundle")
client.run("lock bundle build-order lock1.bundle --json=bo.json")
order = client.load("bo.json")
print(order)
order = json.loads(order)
assert order == [
["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"],
["app1/0.1@#5af607abc205b47375f485a98abc3b38"]
]