/
graph_builder.py
477 lines (414 loc) · 25.1 KB
/
graph_builder.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
import time
from conans.client.conanfile.configure import run_configure_method
from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST, CONTEXT_BUILD
from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod,
conanfile_exception_formatter, ConanInvalidConfiguration)
from conans.model.conan_file import get_env_context_manager
from conans.model.ref import ConanFileReference
from conans.model.requires import Requirements, Requirement
from conans.util.log import logger
class DepsGraphBuilder(object):
"""
This is a summary, in pseudo-code of the execution and structure of the graph
resolution algorithm
load_graph(root_node)
init root_node
expand_node(root_node)
# 1. Evaluate requirements(), overrides, and version ranges
get_node_requirements(node)
node.conanfile.requirements() # call the requirements()
resolve_cached_alias(node.conanfile.requires) # replace cached alias
update_requires_from_downstream(down_requires) # process overrides
resolve_ranges(node) # resolve version-ranges
resolve_cached_alias(node.conanfile.requires) # replace cached alias again
# 2. Process each requires of this node
for req in node.conanfile.requires:
expand_require(req)
if req.name not in graph: # New node
new_node = create_new_node(req) # fetch and load conanfile.py
if alias => create_new_node(alias) # recurse alias
expand_node(new_node) # recursion
else: # Node exists, closing diamond
resolve_cached_alias(req)
check_conflicts(req) # diamonds can cause conflicts
if need_recurse: # check for conflicts upstream
expand_node(previous_node) # recursion
"""
def __init__(self, proxy, output, loader, resolver, recorder):
self._proxy = proxy
self._output = output
self._loader = loader
self._resolver = resolver
self._recorder = recorder
def load_graph(self, root_node, check_updates, update, remotes, profile_host, profile_build,
graph_lock=None):
check_updates = check_updates or update
initial = graph_lock.initial_counter if graph_lock else None
dep_graph = DepsGraph(initial_node_id=initial)
# compute the conanfile entry point for this dependency graph
root_node.public_closure.add(root_node)
root_node.public_deps.add(root_node)
root_node.transitive_closure[root_node.name] = root_node
if profile_build:
root_node.conanfile.settings_build = profile_build.processed_settings.copy()
root_node.conanfile.settings_target = None
dep_graph.add_node(root_node)
# enter recursive computation
t1 = time.time()
self._expand_node(root_node, dep_graph, Requirements(), None, None, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
logger.debug("GRAPH: Time to load deps %s" % (time.time() - t1))
return dep_graph
def extend_build_requires(self, graph, node, build_requires_refs, check_updates, update,
remotes, profile_host, profile_build, graph_lock):
# The options that will be defined in the node will be the real options values that have
# been already propagated downstream from the dependency graph. This will override any
# other possible option in the build_requires dependency graph. This means that in theory
# an option conflict while expanding the build_requires is impossible
node.conanfile.build_requires_options.clear_unscoped_options()
new_options = node.conanfile.build_requires_options._reqs_options
new_reqs = Requirements()
conanfile = node.conanfile
scope = conanfile.display_name
build_requires = []
for ref, context in build_requires_refs:
r = Requirement(ref)
r.build_require = True
r.build_require_context = context
r.force_host_context = getattr(ref, "force_host_context", False)
build_requires.append(r)
if graph_lock:
graph_lock.pre_lock_node(node)
# TODO: Add info about context?
graph_lock.lock_node(node, build_requires, build_requires=True)
for require in build_requires:
self._resolve_alias(node, require, graph, update, update, remotes)
self._resolve_ranges(graph, build_requires, scope, update, remotes)
for br in build_requires:
context_switch = bool(br.build_require_context == CONTEXT_BUILD)
populate_settings_target = context_switch # Avoid 'settings_target' for BR-host
self._expand_require(br, node, graph, check_updates, update,
remotes, profile_host, profile_build, new_reqs, new_options,
graph_lock, context_switch=context_switch,
populate_settings_target=populate_settings_target)
new_nodes = set(n for n in graph.nodes if n.package_id is None)
# This is to make sure that build_requires have precedence over the normal requires
node.public_closure.sort(key_fn=lambda x: x not in new_nodes)
return new_nodes
def _expand_node(self, node, graph, down_reqs, down_ref, down_options, check_updates, update,
remotes, profile_host, profile_build, graph_lock):
""" expands the dependencies of the node, recursively
param node: Node object to be expanded in this step
down_reqs: the Requirements as coming from downstream, which can overwrite current
values
param down_ref: ConanFileReference of who is depending on current node for this expansion
"""
# basic node configuration: calling configure() and requirements() and version-ranges
new_options, new_reqs = self._get_node_requirements(node, graph, down_ref, down_options,
down_reqs, graph_lock, update, remotes)
# Expand each one of the current requirements
for require in node.conanfile.requires.values():
if require.override:
continue
self._expand_require(require, node, graph, check_updates, update, remotes, profile_host,
profile_build, new_reqs, new_options, graph_lock,
context_switch=False)
def _resolve_ranges(self, graph, requires, consumer, update, remotes):
for require in requires:
if require.locked_id: # if it is locked, nothing to resolved
continue
self._resolver.resolve(require, consumer, update, remotes)
self._resolve_cached_alias(requires, graph)
@staticmethod
def _resolve_cached_alias(requires, graph):
if graph.aliased:
for require in requires:
alias = graph.aliased.get(require.ref)
if alias:
require.ref = alias
def _resolve_alias(self, node, require, graph, check_updates, update, remotes):
alias = require.alias
if alias is None:
return
# First try cached
cached = graph.new_aliased.get(alias)
if cached is not None:
while True:
new_cached = graph.new_aliased.get(cached)
if new_cached is None:
break
else:
cached = new_cached
require.ref = cached
return
while alias is not None:
# if not cached, then resolve
try:
result = self._proxy.get_recipe(alias, check_updates, update, remotes, self._recorder)
conanfile_path, recipe_status, remote, new_ref = result
except ConanException as e:
raise e
dep_conanfile = self._loader.load_basic(conanfile_path)
try:
pointed_ref = ConanFileReference.loads(dep_conanfile.alias)
except Exception as e:
raise ConanException("Alias definition error in {}: {}".format(alias, str(e)))
# UPDATE THE REQUIREMENT!
require.ref = require.range_ref = pointed_ref
graph.new_aliased[alias] = pointed_ref # Caching the alias
new_req = Requirement(pointed_ref) # FIXME: Ugly temp creation just for alias check
alias = new_req.alias
def _get_node_requirements(self, node, graph, down_ref, down_options, down_reqs, graph_lock,
update, remotes):
""" compute the requirements of a node, evaluating requirements(), propagating
the downstream requirements and overrides and solving version-ranges
"""
# basic node configuration: calling configure() and requirements()
if graph_lock:
graph_lock.pre_lock_node(node)
new_options = self._config_node(node, down_ref, down_options)
for require in node.conanfile.requires.values():
self._resolve_alias(node, require, graph, update, update, remotes)
# Alias that are cached should be replaced here, bc next requires.update() will warn if not
self._resolve_cached_alias(node.conanfile.requires.values(), graph)
if graph_lock: # No need to evaluate, they are hardcoded in lockfile
graph_lock.lock_node(node, node.conanfile.requires.values())
# propagation of requirements can be necessary if some nodes are not locked
new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref)
# if there are version-ranges, resolve them before expanding each of the requirements
# Resolve possible version ranges of the current node requirements
# new_reqs is a shallow copy of what is propagated upstream, so changes done by the
# RangeResolver are also done in new_reqs, and then propagated!
conanfile = node.conanfile
scope = conanfile.display_name
self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes)
if not hasattr(conanfile, "_conan_evaluated_requires"):
conanfile._conan_evaluated_requires = conanfile.requires.copy()
elif conanfile.requires != conanfile._conan_evaluated_requires:
raise ConanException("%s: Incompatible requirements obtained in different "
"evaluations of 'requirements'\n"
" Previous requirements: %s\n"
" New requirements: %s"
% (scope, list(conanfile._conan_evaluated_requires.values()),
list(conanfile.requires.values())))
return new_options, new_reqs
def _expand_require(self, require, node, graph, check_updates, update, remotes, profile_host,
profile_build, new_reqs, new_options, graph_lock, context_switch,
populate_settings_target=True):
# Handle a requirement of a node. There are 2 possibilities
# node -(require)-> new_node (creates a new node in the graph)
# node -(require)-> previous (creates a diamond with a previously existing node)
# If the required is found in the node ancestors a loop is being closed
context = CONTEXT_BUILD if context_switch else node.context
name = require.ref.name # TODO: allow bootstrapping, use references instead of names
if node.ancestors.get(name, context) or (name == node.name and context == node.context):
raise ConanException("Loop detected in context %s: '%s' requires '%s'"
" which is an ancestor too" % (context, node.ref, require.ref))
# If the requirement is found in the node public dependencies, it is a diamond
previous = node.public_deps.get(name, context=context)
previous_closure = node.public_closure.get(name, context=context)
# build_requires and private will create a new node if it is not in the current closure
if not previous or ((require.build_require or require.private) and not previous_closure):
# new node, must be added and expanded (node -> new_node)
new_node = self._create_new_node(node, graph, require, check_updates, update,
remotes, profile_host, profile_build, graph_lock,
context_switch=context_switch,
populate_settings_target=populate_settings_target)
# The closure of a new node starts with just itself
new_node.public_closure.add(new_node)
new_node.transitive_closure[new_node.name] = new_node
# The new created node is connected to the parent one
node.connect_closure(new_node)
if require.private or require.build_require:
# If the requirement is private (or build_require), a new public_deps is defined
# the new_node doesn't propagate downstream the "node" consumer, so its public_deps
# will be a copy of the node.public_closure, i.e. it can only cause conflicts in the
# new_node.public_closure.
new_node.public_deps.assign(node.public_closure)
new_node.public_deps.add(new_node)
else:
node.transitive_closure[new_node.name] = new_node
# Normal requires propagate and can conflict with the parent "node.public_deps" too
new_node.public_deps.assign(node.public_deps)
new_node.public_deps.add(new_node)
# All the dependents of "node" are also connected now to "new_node"
for dep_node in node.inverse_closure:
dep_node.connect_closure(new_node)
# RECURSION, keep expanding (depth-first) the new node
self._expand_node(new_node, graph, new_reqs, node.ref, new_options, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
if not require.private and not require.build_require:
for name, n in new_node.transitive_closure.items():
node.transitive_closure[name] = n
else: # a public node already exist with this name
self._resolve_cached_alias([require], graph)
# As we are closing a diamond, there can be conflicts. This will raise if conflicts
conflict = self._conflicting_references(previous, require.ref, node.ref)
if conflict: # It is possible to get conflict from alias, try to resolve it
self._resolve_recipe(node, graph, require, check_updates,
update, remotes, profile_host, graph_lock)
# Maybe it was an ALIAS, so we can check conflict again
conflict = self._conflicting_references(previous, require.ref, node.ref)
if conflict:
raise ConanException(conflict)
# Add current ancestors to the previous node and upstream deps
for n in previous.public_closure:
n.ancestors.add(node)
for item in node.ancestors:
n.ancestors.add(item)
node.connect_closure(previous)
graph.add_edge(node, previous, require)
if not require.private and not require.build_require:
for name, n in previous.transitive_closure.items():
node.transitive_closure[name] = n
# All the upstream dependencies (public_closure) of the previously existing node
# now will be also connected to the node and to all its dependants
for n in previous.transitive_closure.values():
node.connect_closure(n)
for dep_node in node.inverse_closure:
dep_node.connect_closure(n)
# Recursion is only necessary if the inputs conflict with the current "previous"
# configuration of upstream versions and options
# recursion can stop if there is a graph_lock not relaxed
lock_recurse = not (graph_lock and not graph_lock.relaxed)
if lock_recurse and self._recurse(previous.public_closure, new_reqs, new_options,
previous.context):
self._expand_node(previous, graph, new_reqs, node.ref, new_options, check_updates,
update, remotes, profile_host, profile_build, graph_lock)
@staticmethod
def _conflicting_references(previous, new_ref, consumer_ref=None):
if previous.ref.copy_clear_rev() != new_ref.copy_clear_rev():
if consumer_ref:
return ("Conflict in %s:\n"
" '%s' requires '%s' while '%s' requires '%s'.\n"
" To fix this conflict you need to override the package '%s' "
"in your root package."
% (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src,
previous.ref, new_ref.name))
return "Unresolvable conflict between {} and {}".format(previous.ref, new_ref)
# Computed node, if is Editable, has revision=None
# If new_ref.revision is None we cannot assume any conflict, the user hasn't specified
# a revision, so it's ok any previous_ref
if previous.ref.revision and new_ref.revision and previous.ref.revision != new_ref.revision:
if consumer_ref:
raise ConanException("Conflict in %s\n"
" Different revisions of %s has been requested"
% (consumer_ref, new_ref))
return True
return False
def _recurse(self, closure, new_reqs, new_options, context):
""" For a given closure, if some requirements or options coming from downstream
is incompatible with the current closure, then it is necessary to recurse
then, incompatibilities will be raised as usually"""
for req in new_reqs.values():
n = closure.get(req.ref.name, context=context)
if n and self._conflicting_references(n, req.ref):
return True
for pkg_name, options_values in new_options.items():
n = closure.get(pkg_name, context=context)
if n:
options = n.conanfile.options
for option, value in options_values.items():
if getattr(options, option) != value:
return True
return False
@staticmethod
def _config_node(node, down_ref, down_options):
""" update settings and option in the current ConanFile, computing actual
requirement values, cause they can be overridden by downstream requires
param settings: dict of settings values => {"os": "windows"}
"""
conanfile, ref = node.conanfile, node.ref
try:
run_configure_method(conanfile, down_options, down_ref, ref)
with get_env_context_manager(conanfile, without_python=True):
# Update requirements (overwrites), computing new upstream
if hasattr(conanfile, "requirements"):
# If re-evaluating the recipe, in a diamond graph, with different options,
# it could happen that one execution path of requirements() defines a package
# and another one a different package raising Duplicate dependency error
# Or the two consecutive calls, adding 2 different dependencies for the 2 paths
# So it is necessary to save the "requires" state and restore it before a second
# execution of requirements(). It is a shallow copy, if first iteration is
# RequireResolve'd or overridden, the inner requirements are modified
if not hasattr(conanfile, "_conan_original_requires"):
conanfile._conan_original_requires = conanfile.requires.copy()
else:
conanfile.requires = conanfile._conan_original_requires.copy()
with conanfile_exception_formatter(str(conanfile), "requirements"):
conanfile.requirements()
new_options = conanfile.options.deps_package_values
except ConanExceptionInUserConanfileMethod:
raise
except ConanException as e:
raise ConanException("%s: %s" % (ref or "Conanfile", str(e)))
except Exception as e:
raise ConanException(e)
return new_options
def _resolve_recipe(self, current_node, dep_graph, requirement, check_updates,
update, remotes, profile, graph_lock, original_ref=None):
try:
result = self._proxy.get_recipe(requirement.ref, check_updates, update,
remotes, self._recorder)
except ConanException as e:
if current_node.ref:
self._output.error("Failed requirement '%s' from '%s'"
% (requirement.ref, current_node.conanfile.display_name))
raise e
conanfile_path, recipe_status, remote, new_ref = result
locked_id = requirement.locked_id
lock_py_requires = graph_lock.python_requires(locked_id) if locked_id is not None else None
dep_conanfile = self._loader.load_conanfile(conanfile_path, profile, ref=requirement.ref,
lock_python_requires=lock_py_requires)
if recipe_status == RECIPE_EDITABLE:
dep_conanfile.in_local_cache = False
dep_conanfile.develop = True
if getattr(dep_conanfile, "alias", None):
new_ref_norev = new_ref.copy_clear_rev()
pointed_ref = ConanFileReference.loads(dep_conanfile.alias)
dep_graph.aliased[new_ref_norev] = pointed_ref # Caching the alias
requirement.ref = pointed_ref
if original_ref: # So transitive alias resolve to the latest in the chain
dep_graph.aliased[original_ref] = pointed_ref
return self._resolve_recipe(current_node, dep_graph, requirement, check_updates,
update, remotes, profile, graph_lock, original_ref)
return new_ref, dep_conanfile, recipe_status, remote, locked_id
def _create_new_node(self, current_node, dep_graph, requirement, check_updates,
update, remotes, profile_host, profile_build, graph_lock, context_switch,
populate_settings_target):
# If there is a context_switch, it is because it is a BR-build
if context_switch:
profile = profile_build
context = CONTEXT_BUILD
else:
profile = profile_host if current_node.context == CONTEXT_HOST else profile_build
context = current_node.context
result = self._resolve_recipe(current_node, dep_graph, requirement, check_updates, update,
remotes, profile, graph_lock)
new_ref, dep_conanfile, recipe_status, remote, locked_id = result
# Assign the profiles depending on the context
if profile_build: # Keep existing behavior (and conanfile members) if no profile_build
dep_conanfile.settings_build = profile_build.processed_settings.copy()
if not context_switch:
if populate_settings_target:
dep_conanfile.settings_target = current_node.conanfile.settings_target
else:
dep_conanfile.settings_target = None
else:
if current_node.context == CONTEXT_HOST:
dep_conanfile.settings_target = profile_host.processed_settings.copy()
else:
dep_conanfile.settings_target = profile_build.processed_settings.copy()
logger.debug("GRAPH: new_node: %s" % str(new_ref))
new_node = Node(new_ref, dep_conanfile, context=context)
new_node.revision_pinned = requirement.ref.revision is not None
new_node.recipe = recipe_status
new_node.remote = remote
# Ancestors are a copy of the parent, plus the parent itself
new_node.ancestors.assign(current_node.ancestors)
new_node.ancestors.add(current_node)
if locked_id is not None:
new_node.id = locked_id
dep_graph.add_node(new_node)
dep_graph.add_edge(current_node, new_node, requirement)
return new_node