/
graph_binaries.py
319 lines (282 loc) · 14.7 KB
/
graph_binaries.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
import os
from conans.client.graph.graph import (BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_MISSING,
BINARY_SKIP, BINARY_UPDATE,
RECIPE_EDITABLE, BINARY_EDITABLE,
RECIPE_CONSUMER, RECIPE_VIRTUAL)
from conans.errors import NoRemoteAvailable, NotFoundException, conanfile_exception_formatter
from conans.model.info import ConanInfo, PACKAGE_ID_UNKNOWN
from conans.model.manifest import FileTreeManifest
from conans.model.ref import PackageReference
from conans.util.files import is_dirty, rmdir
class GraphBinariesAnalyzer(object):
def __init__(self, cache, output, remote_manager):
self._cache = cache
self._out = output
self._remote_manager = remote_manager
# These are the nodes with pref (not including PREV) that have been evaluated
self._evaluated = {} # {pref: [nodes]}
@staticmethod
def _check_update(upstream_manifest, package_folder, output, node):
read_manifest = FileTreeManifest.load(package_folder)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
node.update_manifest = upstream_manifest
return True
else:
output.warn("Current package is newer than remote upstream one")
@staticmethod
def _evaluate_build(node, build_mode):
ref, conanfile = node.ref, node.conanfile
with_deps_to_build = False
# For cascade mode, we need to check also the "modified" status of the lockfile if exists
# modified nodes have already been built, so they shouldn't be built again
if build_mode.cascade and not (node.graph_lock_node and node.graph_lock_node.modified):
for dep in node.dependencies:
dep_node = dep.dst
if (dep_node.binary == BINARY_BUILD or
(dep_node.graph_lock_node and dep_node.graph_lock_node.modified)):
with_deps_to_build = True
break
if build_mode.forced(conanfile, ref, with_deps_to_build):
conanfile.output.info('Forced build from source')
node.binary = BINARY_BUILD
node.prev = None
return True
def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, package_folder, pref):
# Check if dirty, to remove it
with package_layout.package_lock(pref):
assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code"
if is_dirty(package_folder):
node.conanfile.output.warn("Package is corrupted, removing folder: %s"
% package_folder)
rmdir(package_folder) # Do not remove if it is EDITABLE
return
if self._cache.config.revisions_enabled:
metadata = package_layout.load_metadata()
rec_rev = metadata.packages[pref.id].recipe_revision
if rec_rev and rec_rev != node.ref.revision:
node.conanfile.output.warn("The package {} doesn't belong to the installed "
"recipe revision, removing folder".format(pref))
rmdir(package_folder)
return metadata
def _evaluate_cache_pkg(self, node, package_layout, pref, metadata, remote, remotes, update,
package_folder):
if update:
output = node.conanfile.output
if remote:
try:
tmp = self._remote_manager.get_package_manifest(pref, remote)
upstream_manifest, pref = tmp
except NotFoundException:
output.warn("Can't update, no package in remote")
except NoRemoteAvailable:
output.warn("Can't update, no remote defined")
else:
if self._check_update(upstream_manifest, package_folder, output, node):
node.binary = BINARY_UPDATE
node.prev = pref.revision # With revision
elif remotes:
pass # Current behavior: no remote explicit or in metadata, do not update
else:
output.warn("Can't update, no remote defined")
if not node.binary:
node.binary = BINARY_CACHE
metadata = metadata or package_layout.load_metadata()
node.prev = metadata.packages[pref.id].revision
assert node.prev, "PREV for %s is None: %s" % (str(pref), metadata.dumps())
def _evaluate_remote_pkg(self, node, pref, remote, remotes, build_mode):
remote_info = None
if remote:
try:
remote_info, pref = self._remote_manager.get_package_info(pref, remote)
except NotFoundException:
pass
except Exception:
node.conanfile.output.error("Error downloading binary package: '{}'".format(pref))
raise
# If the "remote" came from the registry but the user didn't specified the -r, with
# revisions iterate all remotes
if not remote or (not remote_info and self._cache.config.revisions_enabled):
for r in remotes.values():
try:
remote_info, pref = self._remote_manager.get_package_info(pref, r)
except NotFoundException:
pass
else:
if remote_info:
remote = r
break
if remote_info:
node.binary = BINARY_DOWNLOAD
node.prev = pref.revision
recipe_hash = remote_info.recipe_hash
else:
recipe_hash = None
if build_mode.allowed(node.conanfile):
node.binary = BINARY_BUILD
else:
node.binary = BINARY_MISSING
node.prev = None
return recipe_hash, remote
def _evaluate_is_cached(self, node, pref):
previous_nodes = self._evaluated.get(pref)
if previous_nodes:
previous_nodes.append(node)
previous_node = previous_nodes[0]
# The previous node might have been skipped, but current one not necessarily
# keep the original node.binary value (before being skipped), and if it will be
# defined as SKIP again by self._handle_private(node) if it is really private
if previous_node.binary == BINARY_SKIP:
node.binary = previous_node.binary_non_skip
else:
node.binary = previous_node.binary
node.binary_remote = previous_node.binary_remote
node.prev = previous_node.prev
return True
self._evaluated[pref] = [node]
def _evaluate_node(self, node, build_mode, update, remotes):
assert node.binary is None, "Node.binary should be None"
assert node.package_id is not None, "Node.package_id shouldn't be None"
assert node.package_id != PACKAGE_ID_UNKNOWN, "Node.package_id shouldn't be Unknown"
assert node.prev is None, "Node.prev should be None"
if node.package_id == PACKAGE_ID_UNKNOWN:
node.binary = BINARY_MISSING
return
ref, conanfile = node.ref, node.conanfile
# If it has lock
locked = node.graph_lock_node
if locked and locked.pref.id == node.package_id:
pref = locked.pref # Keep the locked with PREV
else:
assert node.prev is None, "Non locked node shouldn't have PREV in evaluate_node"
pref = PackageReference(ref, node.package_id)
# Check that this same reference hasn't already been checked
if self._evaluate_is_cached(node, pref):
return
if node.recipe == RECIPE_EDITABLE:
node.binary = BINARY_EDITABLE # TODO: PREV?
return
if self._evaluate_build(node, build_mode):
return
package_layout = self._cache.package_layout(pref.ref, short_paths=conanfile.short_paths)
package_folder = package_layout.package(pref)
metadata = self._evaluate_clean_pkg_folder_dirty(node, package_layout, package_folder, pref)
remote = remotes.selected
if not remote:
# If the remote_name is not given, follow the binary remote, or the recipe remote
# If it is defined it won't iterate (might change in conan2.0)
metadata = metadata or package_layout.load_metadata()
remote_name = metadata.packages[pref.id].remote or metadata.recipe.remote
remote = remotes.get(remote_name)
if os.path.exists(package_folder): # Binary already in cache, check for updates
self._evaluate_cache_pkg(node, package_layout, pref, metadata, remote, remotes, update,
package_folder)
recipe_hash = None
else: # Binary does NOT exist locally
# Returned remote might be different than the passed one if iterating remotes
recipe_hash, remote = self._evaluate_remote_pkg(node, pref, remote, remotes, build_mode)
if build_mode.outdated:
if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE):
if node.binary == BINARY_UPDATE:
info, pref = self._remote_manager.get_package_info(pref, remote)
recipe_hash = info.recipe_hash
elif node.binary == BINARY_CACHE:
recipe_hash = ConanInfo.load_from_package(package_folder).recipe_hash
local_recipe_hash = package_layout.recipe_manifest().summary_hash
if local_recipe_hash != recipe_hash:
conanfile.output.info("Outdated package!")
node.binary = BINARY_BUILD
node.prev = None
else:
conanfile.output.info("Package is up to date")
node.binary_remote = remote
@staticmethod
def _propagate_options(node):
# TODO: This has to be moved to the graph computation, not here in the BinaryAnalyzer
# as this is the graph model
conanfile = node.conanfile
neighbors = node.neighbors()
transitive_reqs = set() # of PackageReference, avoid duplicates
for neighbor in neighbors:
ref, nconan = neighbor.ref, neighbor.conanfile
transitive_reqs.add(neighbor.pref)
transitive_reqs.update(nconan.info.requires.refs())
conanfile.options.propagate_downstream(ref, nconan.info.full_options)
# Might be never used, but update original requirement, just in case
conanfile.requires[ref.name].ref = ref
# There might be options that are not upstream, backup them, might be for build-requires
conanfile.build_requires_options = conanfile.options.values
conanfile.options.clear_unused(transitive_reqs)
conanfile.options.freeze()
@staticmethod
def _compute_package_id(node, default_package_id_mode):
"""
Compute the binary package ID of this node
:param node: the node to compute the package-ID
:param default_package_id_mode: configuration of the package-ID mode
"""
# TODO Conan 2.0. To separate the propagation of the graph (options) of the package-ID
# A bit risky to be done now
conanfile = node.conanfile
neighbors = node.neighbors()
direct_reqs = [] # of PackageReference
indirect_reqs = set() # of PackageReference, avoid duplicates
for neighbor in neighbors:
ref, nconan = neighbor.ref, neighbor.conanfile
direct_reqs.append(neighbor.pref)
indirect_reqs.update(nconan.info.requires.refs())
# Make sure not duplicated
indirect_reqs.difference_update(direct_reqs)
conanfile.info = ConanInfo.create(conanfile.settings.values,
conanfile.options.values,
direct_reqs,
indirect_reqs,
default_package_id_mode=default_package_id_mode)
# Once we are done, call package_id() to narrow and change possible values
with conanfile_exception_formatter(str(conanfile), "package_id"):
conanfile.package_id()
info = conanfile.info
node.package_id = info.package_id()
def _handle_private(self, node):
if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE, BINARY_SKIP):
private_neighbours = node.private_neighbors()
for neigh in private_neighbours:
if not neigh.private:
continue
# Current closure contains own node to be skipped
for n in neigh.public_closure.values():
if n.private:
# store the binary origin before being overwritten by SKIP
n.binary_non_skip = n.binary
n.binary = BINARY_SKIP
self._handle_private(n)
def evaluate_graph(self, deps_graph, build_mode, update, remotes):
default_package_id_mode = self._cache.config.default_package_id_mode
for node in deps_graph.ordered_iterate():
self._propagate_options(node)
self._compute_package_id(node, default_package_id_mode)
if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
continue
if node.package_id == PACKAGE_ID_UNKNOWN:
assert node.binary is None
continue
self._evaluate_node(node, build_mode, update, remotes)
self._handle_private(node)
def reevaluate_node(self, node, remotes, build_mode, update):
assert node.binary is None
output = node.conanfile.output
node._package_id = None # Invalidate it, so it can be re-computed
default_package_id_mode = self._cache.config.default_package_id_mode
output.info("Unknown binary for %s, computing updated ID" % str(node.ref))
self._compute_package_id(node, default_package_id_mode)
output.info("Updated ID: %s" % node.package_id)
if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
return
assert node.package_id != PACKAGE_ID_UNKNOWN
self._evaluate_node(node, build_mode, update, remotes)
output.info("Binary for updated ID from: %s" % node.binary)
if node.binary == BINARY_BUILD:
output.info("Binary for the updated ID has to be built")
else:
output.info("Binary for the updated ID from: %s" % node.binary)