forked from conan-io/conan
/
lock_bundle.py
178 lines (156 loc) · 7.49 KB
/
lock_bundle.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
import json
import os
from conans.errors import ConanException
from conans.model.graph_lock import GraphLockFile
from conans.util.files import load, save
class LockBundle(object):
""" aggregated view of multiple lockfiles, coming from different configurations and also
different products, that is, completely independent graphs, that could have incompatible
versions. This bundle is not useful for restoring a state for a configuration, cannot be
used to lock a graph, but it it useful for computing the build order in CI for multiple
products. The format is a json with: For every reference, for each "package_id" for that
reference, list the lockfiles that contain such reference:package_id and the node indexes
inside that lockfile
lock_bundle": {
"app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2": {
"packages": [
{
"package_id": "3bcd6800847f779e0883ee91b411aad9ddd8e83c",
"lockfiles": {
"app1_windows.lock": [
"1"
]
},
"prev": null,
"modified": null
},
],
"requires": [
"pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20",
"tool/0.1@#f096d7d54098b7ad7012f9435d9c33f3"
]
},
"""
def __init__(self):
self._nodes = {}
@staticmethod
def create(lockfiles, revisions_enabled, cwd):
def ref_convert(r):
# Necessary so "build-order" output is usable by install
if "@" not in r:
if "#" in r:
r = r.replace("#", "@#")
else:
r += "@"
return r
result = LockBundle()
for lockfile_name in lockfiles:
lockfile_abs = os.path.normpath(os.path.join(cwd, lockfile_name))
lockfile = GraphLockFile.load(lockfile_abs, revisions_enabled)
lock = lockfile.graph_lock
for id_, node in lock.nodes.items():
ref_str = node.ref.full_str()
ref_str = ref_convert(ref_str)
ref_node = result._nodes.setdefault(ref_str, {})
packages_node = ref_node.setdefault("packages", [])
# Find existing package_id in the list of packages
# This is the equivalent of a setdefault over a dict, but on a list
for pkg in packages_node:
if pkg["package_id"] == node.package_id:
pid_node = pkg
break
else:
pid_node = {"package_id": node.package_id}
packages_node.append(pid_node)
ids = pid_node.setdefault("lockfiles", {})
# TODO: Add check that this prev is always the same
pid_node["prev"] = node.prev
pid_node["modified"] = node.modified
ids.setdefault(lockfile_name, []).append(id_)
total_requires = node.requires + node.build_requires
for require in total_requires:
require_node = lock.nodes[require]
ref = require_node.ref.full_str()
ref = ref_convert(ref)
requires = ref_node.setdefault("requires", [])
if ref not in requires:
requires.append(ref)
return result
def dumps(self):
return json.dumps({"lock_bundle": self._nodes}, indent=4)
def loads(self, text):
nodes_json = json.loads(text)
self._nodes = nodes_json["lock_bundle"]
def build_order(self):
# First do a topological order by levels, the ids of the nodes are stored
levels = []
opened = list(self._nodes.keys())
while opened:
current_level = []
closed = []
for o in opened:
node = self._nodes[o]
requires = node.get("requires", [])
if not any(n in opened for n in requires): # Doesn't have an open requires
# iterate all packages to see if some has prev=null
if any(pkg["prev"] is None for pkg in node["packages"]):
current_level.append(o)
closed.append(o)
if current_level:
current_level.sort()
levels.append(current_level)
# now initialize new level
opened = set(opened).difference(closed)
return levels
@staticmethod
def update_bundle(bundle_path, revisions_enabled):
""" Update both the bundle information as well as every individual lockfile, from the
information that was modified in the individual lockfile. At the end, all lockfiles will
have the same PREV for the binary of same package_id
"""
bundle = LockBundle()
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
# Each node in bundle belongs to a "ref", and contains lockinfo for every package_id
for pkg in node["packages"]:
# Each package_id contains information of multiple lockfiles
# First, compute the modified PREV from all lockfiles
prev = modified = prev_lockfile = None
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
# Make sure the PREV from lockfiles is consistent, it cannot be different
lock_prev = graph_lock.nodes[node_id].prev
if prev is None:
prev = lock_prev
prev_lockfile = lockfile
modified = graph_lock.nodes[node_id].modified
elif lock_prev is not None and prev != lock_prev:
ref = graph_lock.nodes[node_id].ref
msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format(
ref, prev_lockfile, prev, lockfile, lock_prev)
raise ConanException(msg)
pkg["prev"] = prev
pkg["modified"] = modified
# Then, update all prev of all config lockfiles
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
if graph_lock.nodes[node_id].prev is None:
graph_lock.nodes[node_id].prev = prev
graph_lock_conf.save(lockfile)
save(bundle_path, bundle.dumps())
@staticmethod
def clean_modified(bundle_path, revisions_enabled):
bundle = LockBundle()
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
for pkg in node["packages"]:
pkg["modified"] = None
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(lockfile, revisions_enabled)
graph_lock_conf.graph_lock.clean_modified()
graph_lock_conf.save(lockfile)
save(bundle_path, bundle.dumps())