forked from conan-io/conan
/
graph_manager.py
478 lines (416 loc) · 25.2 KB
/
graph_manager.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
import fnmatch
import os
from collections import OrderedDict, defaultdict
from conans.client.conanfile.configure import run_configure_method
from conans.client.generators.text import TXTGenerator
from conans.client.graph.build_mode import BuildMode
from conans.client.graph.graph import BINARY_BUILD, Node, CONTEXT_HOST, CONTEXT_BUILD
from conans.client.graph.graph_binaries import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_EDITABLE, \
BINARY_UNKNOWN
from conans.client.graph.graph_builder import DepsGraphBuilder
from conans.errors import ConanException, conanfile_exception_formatter
from conans.model.conan_file import get_env_context_manager
from conans.model.graph_info import GraphInfo
from conans.model.graph_lock import GraphLock, GraphLockFile
from conans.model.ref import ConanFileReference
from conans.paths import BUILD_INFO
from conans.util.files import load
class _RecipeBuildRequires(OrderedDict):
def __init__(self, conanfile, default_context):
super(_RecipeBuildRequires, self).__init__()
# "tool_requires" is an alias for 2.0 compatibility
for require_type in ("build_requires", "tool_requires"):
build_requires = getattr(conanfile, require_type, [])
if not isinstance(build_requires, (list, tuple)):
build_requires = [build_requires]
self._default_context = default_context
for build_require in build_requires:
self.add(build_require, context=self._default_context)
def add(self, build_require, context, force_host_context=False):
if build_require is None:
return
if not isinstance(build_require, ConanFileReference):
build_require = ConanFileReference.loads(build_require)
build_require.force_host_context = force_host_context # Dirty, but will be removed in 2.0
self[(build_require.name, context)] = build_require
def __call__(self, build_require, force_host_context=False):
context = CONTEXT_HOST if force_host_context else self._default_context
self.add(build_require, context, force_host_context)
def __str__(self):
items = ["{} ({})".format(br, ctxt) for (_, ctxt), br in self.items()]
return ", ".join(items)
class GraphManager(object):
def __init__(self, output, cache, remote_manager, loader, proxy, resolver, binary_analyzer):
self._proxy = proxy
self._output = output
self._resolver = resolver
self._cache = cache
self._remote_manager = remote_manager
self._loader = loader
self._binary_analyzer = binary_analyzer
def load_consumer_conanfile(self, conanfile_path, info_folder,
deps_info_required=False, test=False):
"""loads a conanfile for local flow: source, imports, package, build
"""
try:
graph_info = GraphInfo.load(info_folder)
lock_path = os.path.join(info_folder, "conan.lock")
graph_lock_file = GraphLockFile.load(lock_path, self._cache.config.revisions_enabled)
graph_lock = graph_lock_file.graph_lock
self._output.info("Using lockfile: '{}/conan.lock'".format(info_folder))
profile_host = graph_lock_file.profile_host
profile_build = graph_lock_file.profile_build
self._output.info("Using cached profile from lockfile")
except IOError: # Only if file is missing
graph_lock = None
# This is very dirty, should be removed for Conan 2.0 (source() method only)
profile_host = self._cache.default_profile
profile_host.process_settings(self._cache)
profile_build = None
name, version, user, channel = None, None, None, None
else:
name, version, user, channel, _ = graph_info.root
profile_host.process_settings(self._cache, preprocess=False)
# This is the hack of recovering the options from the graph_info
profile_host.options.update(graph_info.options)
if profile_build:
profile_build.process_settings(self._cache, preprocess=False)
if conanfile_path.endswith(".py"):
lock_python_requires = None
if graph_lock and not test: # Only lock python requires if it is not test_package
node_id = graph_lock.get_consumer(graph_info.root)
lock_python_requires = graph_lock.python_requires(node_id)
# The global.conf is necessary for download_cache definition
profile_host.conf.rebase_conf_definition(self._cache.new_config)
conanfile = self._loader.load_consumer(conanfile_path,
profile_host=profile_host,
name=name, version=version,
user=user, channel=channel,
lock_python_requires=lock_python_requires)
if profile_build:
conanfile.settings_build = profile_build.processed_settings.copy()
conanfile.settings_target = None
if test:
conanfile.display_name = "%s (test package)" % str(test)
conanfile.output.scope = conanfile.display_name
conanfile.tested_reference_str = repr(test)
run_configure_method(conanfile, down_options=None, down_ref=None, ref=None)
else:
conanfile = self._loader.load_conanfile_txt(conanfile_path, profile_host=profile_host)
load_deps_info(info_folder, conanfile, required=deps_info_required)
return conanfile
def load_graph(self, reference, create_reference, graph_info, build_mode, check_updates, update,
remotes, recorder, apply_build_requires=True, lockfile_node_id=None,
is_build_require=False, require_overrides=None):
""" main entry point to compute a full dependency graph
"""
profile_host, profile_build = graph_info.profile_host, graph_info.profile_build
graph_lock, root_ref = graph_info.graph_lock, graph_info.root
root_node = self._load_root_node(reference, create_reference, profile_host, graph_lock,
root_ref, lockfile_node_id, is_build_require,
require_overrides)
deps_graph = self._resolve_graph(root_node, profile_host, profile_build, graph_lock,
build_mode, check_updates, update, remotes, recorder,
apply_build_requires=apply_build_requires)
# Run some validations once the graph is built
self._validate_graph_provides(deps_graph)
# THIS IS NECESSARY to store dependencies options in profile, for consumer
# FIXME: This is a hack. Might dissapear if graph for local commands is always recomputed
graph_info.options = root_node.conanfile.options.values
if root_node.ref:
graph_info.root = root_node.ref
if graph_info.graph_lock is None:
graph_info.graph_lock = GraphLock(deps_graph, self._cache.config.revisions_enabled)
return deps_graph
def _load_root_node(self, reference, create_reference, profile_host, graph_lock, root_ref,
lockfile_node_id, is_build_require, require_overrides):
""" creates the first, root node of the graph, loading or creating a conanfile
and initializing it (settings, options) as necessary. Also locking with lockfile
information
"""
profile_host.dev_reference = create_reference # Make sure the created one has develop=True
if isinstance(reference, list): # Install workspace with multiple root nodes
conanfile = self._loader.load_virtual(reference, profile_host, scope_options=False)
# Locking in workspaces not implemented yet
return Node(ref=None, context=CONTEXT_HOST, conanfile=conanfile, recipe=RECIPE_VIRTUAL)
# create (without test_package), install|info|graph|export-pkg <ref>
if isinstance(reference, ConanFileReference):
return self._load_root_direct_reference(reference, graph_lock, profile_host,
lockfile_node_id, is_build_require,
require_overrides)
path = reference # The reference must be pointing to a user space conanfile
if create_reference: # Test_package -> tested reference
ret = self._load_root_test_package(path, create_reference, graph_lock, profile_host,
require_overrides)
return ret
# It is a path to conanfile.py or conanfile.txt
root_node = self._load_root_consumer(path, graph_lock, profile_host, root_ref,
require_overrides)
return root_node
def _load_root_consumer(self, path, graph_lock, profile, ref, require_overrides):
""" load a CONSUMER node from a user space conanfile.py or conanfile.txt
install|info|create|graph <path>
:path full path to a conanfile
:graph_lock: might be None, information of lockfiles
:profile: data to inject to the consumer node: settings, options
:ref: previous reference of a previous command. Can be used for finding itself in
the lockfile, or to initialize
"""
if path.endswith(".py"):
lock_python_requires = None
if graph_lock:
if ref.name is None:
# If the graph_info information is not there, better get what we can from
# the conanfile
# Using load_named() to run set_name() set_version() and get them
# so it can be found by name in the lockfile
conanfile = self._loader.load_named(path, None, None, None, None)
ref = ConanFileReference(ref.name or conanfile.name,
ref.version or conanfile.version,
ref.user, ref.channel, validate=False)
node_id = graph_lock.get_consumer(ref)
lock_python_requires = graph_lock.python_requires(node_id)
conanfile = self._loader.load_consumer(path, profile,
name=ref.name,
version=ref.version,
user=ref.user,
channel=ref.channel,
lock_python_requires=lock_python_requires,
require_overrides=require_overrides)
ref = ConanFileReference(conanfile.name, conanfile.version,
ref.user, ref.channel, validate=False)
root_node = Node(ref, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER, path=path)
else:
conanfile = self._loader.load_conanfile_txt(path, profile, ref=ref,
require_overrides=require_overrides)
root_node = Node(None, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER,
path=path)
if graph_lock: # Find the Node ID in the lock of current root
node_id = graph_lock.get_consumer(root_node.ref)
root_node.id = node_id
return root_node
def _load_root_direct_reference(self, reference, graph_lock, profile, lockfile_node_id,
is_build_require, require_overrides):
""" When a full reference is provided:
install|info|graph <ref> or export-pkg .
:return a VIRTUAL root_node with a conanfile that requires the reference
"""
if not self._cache.config.revisions_enabled and reference.revision is not None:
raise ConanException("Revisions not enabled in the client, specify a "
"reference without revision")
conanfile = self._loader.load_virtual([reference], profile,
is_build_require=is_build_require,
require_overrides=require_overrides)
root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL)
# Build_requires cannot be found as early as this, because there is no require yet
if graph_lock and not is_build_require: # Find the Node ID in the lock of current root
graph_lock.find_require_and_lock(reference, conanfile, lockfile_node_id)
return root_node
def _load_root_test_package(self, path, create_reference, graph_lock, profile,
require_overrides):
""" when a test_package/conanfile.py is provided, together with the reference that is
being created and need to be tested
:return a CONSUMER root_node with a conanfile.py with an injected requires to the
created reference
"""
test = str(create_reference)
# do not try apply lock_python_requires for test_package/conanfile.py consumer
conanfile = self._loader.load_consumer(path, profile, user=create_reference.user,
channel=create_reference.channel,
require_overrides=require_overrides
)
conanfile.display_name = "%s (test package)" % str(test)
conanfile.output.scope = conanfile.display_name
conanfile.tested_reference_str = repr(create_reference)
# Injection of the tested reference
test_type = getattr(conanfile, "test_type", ("requires", ))
if not isinstance(test_type, (list, tuple)):
test_type = (test_type, )
if "explicit" not in test_type: # 2.0 mode, not automatically add the require, always explicit
if "build_requires" in test_type:
if getattr(conanfile, "build_requires", None):
# Injecting the tested reference
existing = conanfile.build_requires
if not isinstance(existing, (list, tuple)):
existing = [existing]
conanfile.build_requires = list(existing) + [create_reference]
else:
conanfile.build_requires = str(create_reference)
if "requires" in test_type:
require = conanfile.requires.get(create_reference.name)
if require:
require.ref = require.range_ref = create_reference
else:
conanfile.requires.add_ref(create_reference)
ref = ConanFileReference(conanfile.name, conanfile.version,
create_reference.user, create_reference.channel, validate=False)
root_node = Node(ref, conanfile, recipe=RECIPE_CONSUMER, context=CONTEXT_HOST, path=path)
if graph_lock:
graph_lock.find_require_and_lock(create_reference, conanfile)
return root_node
def _resolve_graph(self, root_node, profile_host, profile_build, graph_lock, build_mode,
check_updates, update, remotes, recorder, apply_build_requires=True):
build_mode = BuildMode(build_mode, self._output)
deps_graph = self._load_graph(root_node, check_updates, update,
build_mode=build_mode, remotes=remotes,
recorder=recorder,
profile_host=profile_host,
profile_build=profile_build,
apply_build_requires=apply_build_requires,
graph_lock=graph_lock)
version_ranges_output = self._resolver.output
if version_ranges_output:
self._output.success("Version ranges solved")
for msg in version_ranges_output:
self._output.info(" %s" % msg)
self._output.writeln("")
self._resolver.clear_output()
build_mode.report_matches()
return deps_graph
@staticmethod
def _get_recipe_build_requires(conanfile, default_context):
conanfile.build_requires = _RecipeBuildRequires(conanfile, default_context)
conanfile.tool_requires = conanfile.build_requires
class TestRequirements:
def __init__(self, build_requires):
self._build_requires = build_requires
def __call__(self, ref):
self._build_requires(ref, force_host_context=True)
conanfile.test_requires = TestRequirements(conanfile.build_requires)
if hasattr(conanfile, "build_requirements"):
with get_env_context_manager(conanfile):
with conanfile_exception_formatter(str(conanfile), "build_requirements"):
conanfile.build_requirements()
return conanfile.build_requires
def _recurse_build_requires(self, graph, builder, check_updates,
update, build_mode, remotes, profile_build_requires, recorder,
profile_host, profile_build, graph_lock, apply_build_requires=True,
nodes_subset=None, root=None):
"""
:param graph: This is the full dependency graph with all nodes from all recursions
"""
default_context = CONTEXT_BUILD if profile_build else CONTEXT_HOST
self._binary_analyzer.evaluate_graph(graph, build_mode, update, remotes, nodes_subset, root)
if not apply_build_requires:
return
for node in graph.ordered_iterate(nodes_subset):
# Packages with PACKAGE_ID_UNKNOWN might be built in the future, need build requires
if (node.binary not in (BINARY_BUILD, BINARY_EDITABLE, BINARY_UNKNOWN)
and node.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL)):
continue
package_build_requires = self._get_recipe_build_requires(node.conanfile, default_context)
str_ref = str(node.ref)
# Compute the update of the current recipe build_requires when updated with the
# downstream profile-defined build-requires
new_profile_build_requires = []
for pattern, build_requires in profile_build_requires.items():
if node.recipe == RECIPE_VIRTUAL: # Virtual do not get profile build requires
continue
if ((node.recipe == RECIPE_CONSUMER and pattern == "&") or
(node.recipe != RECIPE_CONSUMER and pattern == "&!") or
fnmatch.fnmatch(str_ref, pattern)):
for build_require in build_requires:
br_key = (build_require.name, default_context)
if br_key in package_build_requires: # Override defined
# this is a way to have only one package Name for all versions
# (no conflicts)
# but the dict key is not used at all
package_build_requires[br_key] = build_require
# Profile one or in different context
elif build_require.name != node.name or default_context != node.context:
new_profile_build_requires.append((build_require, default_context))
def _recurse_build_requires(br_list, transitive_build_requires):
nodessub = builder.extend_build_requires(graph, node, br_list, check_updates,
update, remotes, profile_host,
profile_build, graph_lock)
self._recurse_build_requires(graph, builder, check_updates, update, build_mode,
remotes, transitive_build_requires, recorder,
profile_host, profile_build, graph_lock,
nodes_subset=nodessub, root=node)
if package_build_requires:
if default_context == CONTEXT_BUILD:
br_build, br_host = [], []
for (_, ctxt), it in package_build_requires.items():
if ctxt == CONTEXT_BUILD:
br_build.append((it, ctxt))
else:
br_host.append((it, ctxt))
if br_build:
_recurse_build_requires(br_build, profile_build.build_requires)
if br_host:
_recurse_build_requires(br_host, profile_build_requires)
else:
br_all = [(it, ctxt) for (_, ctxt), it in package_build_requires.items()]
_recurse_build_requires(br_all, profile_build_requires)
if new_profile_build_requires:
_recurse_build_requires(new_profile_build_requires, {})
if graph_lock and node.recipe != RECIPE_VIRTUAL:
graph_lock.check_locked_build_requires(node, package_build_requires,
new_profile_build_requires)
def _load_graph(self, root_node, check_updates, update, build_mode, remotes,
recorder, profile_host, profile_build, apply_build_requires,
graph_lock):
assert isinstance(build_mode, BuildMode)
profile_host_build_requires = profile_host.build_requires
builder = DepsGraphBuilder(self._proxy, self._output, self._loader, self._resolver,
recorder)
graph = builder.load_graph(root_node, check_updates, update, remotes, profile_host,
profile_build, graph_lock)
self._recurse_build_requires(graph, builder, check_updates, update, build_mode,
remotes, profile_host_build_requires, recorder, profile_host,
profile_build, graph_lock,
apply_build_requires=apply_build_requires)
# Sort of closures, for linking order
inverse_levels = {n: i for i, level in enumerate(graph.inverse_levels()) for n in level}
for node in graph.nodes:
node.public_closure.pop(node.name, context=node.context)
# List sort is stable, will keep the original order of closure, but prioritize levels
node.public_closure.sort(key_fn=lambda n: inverse_levels[n])
return graph
@staticmethod
def _validate_graph_provides(deps_graph):
# Check that two different nodes are not providing the same (ODR violation)
for node in deps_graph.nodes:
provides = defaultdict(list)
if node.conanfile.provides is not None: # consumer conanfile doesn't initialize
for it in node.conanfile.provides:
provides[it].append(node)
for item in filter(lambda u: u.context == CONTEXT_HOST, node.public_closure):
for it in item.conanfile.provides:
provides[it].append(item)
# Check (and report) if any functionality is provided by several different recipes
conflicts = [it for it, nodes in provides.items() if len(nodes) > 1]
if conflicts:
msg_lines = ["At least two recipes provides the same functionality:"]
for it in conflicts:
nodes_str = "', '".join([n.conanfile.display_name for n in provides[it]])
msg_lines.append(" - '{}' provided by '{}'".format(it, nodes_str))
raise ConanException('\n'.join(msg_lines))
def load_deps_info(current_path, conanfile, required):
def get_forbidden_access_object(field_name):
class InfoObjectNotDefined(object):
def __getitem__(self, item):
raise ConanException("self.%s not defined. If you need it for a "
"local command run 'conan install'" % field_name)
__getattr__ = __getitem__
return InfoObjectNotDefined()
if not current_path:
return
info_file_path = os.path.join(current_path, BUILD_INFO)
try:
deps_cpp_info, deps_user_info, deps_env_info, user_info_build = \
TXTGenerator.loads(load(info_file_path), filter_empty=True)
conanfile.deps_cpp_info = deps_cpp_info
conanfile.deps_user_info = deps_user_info
conanfile.deps_env_info = deps_env_info
if user_info_build:
conanfile.user_info_build = user_info_build
except IOError:
if required:
raise ConanException("%s file not found in %s\nIt is required for this command\n"
"You can generate it using 'conan install'"
% (BUILD_INFO, current_path))
conanfile.deps_cpp_info = get_forbidden_access_object("deps_cpp_info")
conanfile.deps_user_info = get_forbidden_access_object("deps_user_info")
except ConanException:
raise ConanException("Parse error in '%s' file in %s" % (BUILD_INFO, current_path))