forked from conan-io/conan
/
build_info.py
349 lines (287 loc) · 15.1 KB
/
build_info.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
import datetime
import json
import os
from collections import defaultdict, namedtuple
import requests
from six.moves.urllib.parse import urlparse, urljoin
from conans.client.cache.cache import ClientCache
from conans.client.rest import response_to_str
from conans.errors import AuthenticationException, RequestErrorException, ConanException
from conans.model.graph_lock import LOCKFILE_VERSION
from conans.model.ref import ConanFileReference
from conans.paths import ARTIFACTS_PROPERTIES_PUT_PREFIX
from conans.paths import get_conan_user_home
from conans.util.files import save
from conans import __version__
class Artifact(namedtuple('Artifact', ["sha1", "md5", "name", "id", "path"])):
def __hash__(self):
return hash(self.sha1)
def _parse_options(contents):
for line in contents.splitlines():
key, value = line.split("=")
yield "options.{}".format(key), value
def _parse_profile(contents):
import configparser
config = configparser.ConfigParser()
config.read_string(contents)
for section, values in config._sections.items():
for key, value in values.items():
yield "{}.{}".format(section, key), value
class BuildInfoCreator(object):
def __init__(self, output, build_info_file, lockfile, user=None, password=None, apikey=None):
self._build_info_file = build_info_file
self._lockfile = lockfile
self._user = user
self._password = password
self._apikey = apikey
self._output = output
self._conan_cache = ClientCache(os.path.join(get_conan_user_home(), ".conan"), output)
def parse_ref(self, ref):
ref = ConanFileReference.loads(ref, validate=False)
rrev = ref.revision
return {
"name": ref.name,
"version": ref.version,
"user": ref.user,
"channel": ref.channel,
"rrev": rrev,
}
def _get_reference(self, ref):
r = self.parse_ref(ref)
recipe_rev = self._get_recipe_rev(r)
user_channel = self._get_user_channel(r)
return "{name}/{version}{user_channel}{recipe_rev}".format(recipe_rev=recipe_rev,
user_channel=user_channel, **r)
def _get_package_reference(self, ref, pid, prev):
package_rev = "#{}".format(prev) if prev and prev != "0" else ""
return "{reference}:{pid}{package_rev}".format(reference=self._get_reference(ref), pid=pid,
package_rev=package_rev)
def _get_metadata_artifacts(self, metadata, ref_path, use_id=False, name_format="{}",
package_id=None):
ret = {}
need_sources = False
if package_id:
data = metadata.packages[package_id].checksums
else:
data = metadata.recipe.checksums
need_sources = not ("conan_sources.tgz" in data)
for name, value in data.items():
name_or_id = name_format.format(name)
ret[value["sha1"]] = {"md5": value["md5"],
"name": name_or_id if not use_id else None,
"id": name_or_id if use_id else None,
"path": ref_path if not use_id else None}
if need_sources:
remote_name = metadata.recipe.remote
remotes = self._conan_cache.registry.load_remotes()
remote_url = remotes[remote_name].url
parsed_uri = urlparse(remote_url)
base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format(
uri=parsed_uri)
request_url = urljoin(base_url, "{}/conan_sources.tgz".format(ref_path))
if self._user and self._password:
response = requests.get(request_url, auth=(self._user, self._password))
elif self._apikey:
response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey})
else:
response = requests.get(request_url)
if response.status_code == 200:
data = response.json()
ret[data["checksums"]["sha1"]] = {"md5": data["checksums"]["md5"],
"name": "conan_sources.tgz" if not use_id else None,
"id": "conan_sources.tgz" if use_id else None,
"path": ref_path if not use_id else None}
return set([Artifact(k, **v) for k, v in ret.items()])
def _get_repo(self, ref):
metadata = self._get_metadata(ref)
remote_name = metadata.recipe.remote
remotes = self._conan_cache.registry.load_remotes()
remote_url = remotes[remote_name].url
return remote_url.split("/")[-1]
def _get_recipe_rev(self, ref):
return "#{}".format(ref.get("rrev")) if ref.get("rrev") and ref.get("rrev") != "0" else ""
def _get_user_channel(self, ref):
return "@{}/{}".format(ref.get("user"), ref.get("channel")) if ref.get("user") and \
ref.get("channel") else ""
def _get_recipe_path(self, ref):
r = self.parse_ref(ref)
user_channel = self._get_user_channel(r)
path = "{user}/{name}/{version}/{channel}/{rrev}/export".format(
**r) if user_channel else "_/{name}/{version}/_/{rrev}/export".format(**r)
return path
def _get_package_path(self, ref, pid, prev):
r = self.parse_ref(ref)
user_channel = self._get_user_channel(r)
path = "{user}/{name}/{version}/{channel}/{rrev}/package/{pid}/{prev}" if user_channel else \
"_/{name}/{version}/_/{rrev}/package/{pid}/{prev}"
path = path.format(pid=pid, prev=prev, **r)
return path
def _get_metadata(self, ref):
reference = ConanFileReference.loads(self._get_reference(ref))
package_layout = self._conan_cache.package_layout(reference)
metadata = package_layout.load_metadata()
return metadata
def _get_recipe_artifacts(self, ref, is_dependency):
metadata = self._get_metadata(ref)
name_format = "{} :: {{}}".format(self._get_reference(ref)) if is_dependency else "{}"
ref_path = self._get_recipe_path(ref)
return self._get_metadata_artifacts(metadata, ref_path, name_format=name_format, use_id=is_dependency)
def _get_package_artifacts(self, ref, pid, prev, is_dependency):
metadata = self._get_metadata(ref)
name_format = "{} :: {{}}".format(self._get_package_reference(ref, pid, prev)) if is_dependency else "{}"
ref_path = self._get_package_path(ref, pid, prev)
arts = self._get_metadata_artifacts(metadata, ref_path, name_format=name_format,
use_id=is_dependency, package_id=pid)
return arts
def process_lockfile(self):
modules = defaultdict(lambda: {"type": "conan",
"repository": None,
"id": None,
"artifacts": set(),
"dependencies": set()})
def _gather_transitive_recipes(nid, contents):
n = contents["graph_lock"]["nodes"][nid]
artifacts = self._get_recipe_artifacts(n["ref"], is_dependency=True)
for id_node in n.get("requires", []):
artifacts.update(_gather_transitive_recipes(id_node, contents))
for id_node in n.get("build_requires", []):
artifacts.update(_gather_transitive_recipes(id_node, contents))
return artifacts
def _gather_transitive_packages(nid, contents):
n = contents["graph_lock"]["nodes"][nid]
artifacts = self._get_package_artifacts(n["ref"], n["package_id"], n["prev"],
is_dependency=True)
for id_node in n.get("requires", []):
artifacts.update(_gather_transitive_packages(id_node, contents))
for id_node in n.get("build_requires", []):
artifacts.update(_gather_transitive_packages(id_node, contents))
return artifacts
with open(self._lockfile) as json_data:
data = json.load(json_data)
version = data["version"]
if version != LOCKFILE_VERSION:
raise ConanException("This lockfile was created with an incompatible version "
"of Conan. Please update all your Conan clients")
# Gather modules, their artifacts and recursively all required artifacts
for _, node in data["graph_lock"]["nodes"].items():
ref = node["ref"]
pid = node.get("package_id")
prev = node.get("prev")
if node.get("modified"): # Work only on generated nodes
# Create module for the recipe reference
recipe_key = self._get_reference(ref)
repository = self._get_repo(ref)
modules[recipe_key]["id"] = recipe_key
modules[recipe_key]["artifacts"].update(
self._get_recipe_artifacts(ref, is_dependency=False))
modules[recipe_key]["repository"] = repository
# TODO: what about `python_requires`?
# TODO: can we associate any properties to the recipe? Profile/options may
# TODO: be different per lockfile
# Create module for the package_id
package_key = self._get_package_reference(ref, pid, prev)
modules[package_key]["id"] = package_key
modules[package_key]["artifacts"].update(
self._get_package_artifacts(ref, pid, prev, is_dependency=False))
modules[package_key]["repository"] = repository
# Recurse requires
node_ids = node.get("requires", []) + node.get("build_requires", [])
for node_id in node_ids:
modules[recipe_key]["dependencies"].update(_gather_transitive_recipes(node_id,
data))
modules[package_key]["dependencies"].update(_gather_transitive_packages(node_id,
data))
# TODO: Is the recipe a 'dependency' of the package
return modules
def create(self):
properties = self._conan_cache.read_artifacts_properties()
modules = self.process_lockfile()
# Add extra information
ret = {"version": "1.0.1",
"name": properties[ARTIFACTS_PROPERTIES_PUT_PREFIX + "build.name"],
"number": properties[ARTIFACTS_PROPERTIES_PUT_PREFIX + "build.number"],
"type": "GENERIC",
"started": datetime.datetime.utcnow().isoformat().split(".")[0] + ".000Z",
"buildAgent": {"name": "conan", "version": "{}".format(__version__)},
"modules": list(modules.values())}
def dump_custom_types(obj):
if isinstance(obj, set):
artifacts = [{k: v for k, v in o._asdict().items() if v is not None} for o in obj]
return sorted(artifacts, key=lambda u: u.get("name") or u.get("id"))
raise TypeError
save(self._build_info_file, json.dumps(ret, indent=4, default=dump_custom_types))
def create_build_info(output, build_info_file, lockfile, user, password, apikey):
bi = BuildInfoCreator(output, build_info_file, lockfile, user, password, apikey)
bi.create()
def start_build_info(output, build_name, build_number):
paths = ClientCache(os.path.join(get_conan_user_home(), ".conan"), output)
content = ARTIFACTS_PROPERTIES_PUT_PREFIX + "build.name={}\n".format(build_name) + \
ARTIFACTS_PROPERTIES_PUT_PREFIX + "build.number={}\n".format(build_number)
artifact_properties_file = paths.artifacts_properties_path
try:
save(artifact_properties_file, content)
except Exception:
raise ConanException("Can't write properties file in %s" % artifact_properties_file)
def stop_build_info(output):
paths = ClientCache(os.path.join(get_conan_user_home(), ".conan"), output)
artifact_properties_file = paths.artifacts_properties_path
try:
save(artifact_properties_file, "")
except Exception:
raise ConanException("Can't write properties file in %s" % artifact_properties_file)
def publish_build_info(build_info_file, url, user, password, apikey):
with open(build_info_file) as json_data:
parsed_uri = urlparse(url)
request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format(uri=parsed_uri)
if user and password:
response = requests.put(request_url, headers={"Content-Type": "application/json"},
data=json_data, auth=(user, password))
elif apikey:
response = requests.put(request_url, headers={"Content-Type": "application/json",
"X-JFrog-Art-Api": apikey},
data=json_data)
else:
response = requests.put(request_url)
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
elif response.status_code != 204:
raise RequestErrorException(response_to_str(response))
def find_module(build_info, module_id):
for it in build_info["modules"]:
if it["id"] == module_id:
return it
new_module = {"id": module_id, "artifacts": [], "dependencies": []}
build_info["modules"].append(new_module)
return new_module
def merge_artifacts(lhs, rhs, key, cmp_key):
ret = {it[cmp_key]: it for it in lhs[key]}
for art in rhs[key]:
art_cmp_key = art[cmp_key]
if art_cmp_key in ret:
assert art[cmp_key] == ret[art_cmp_key][cmp_key], \
"({}) {} != {} for sha1={}".format(cmp_key, art[cmp_key], ret[art_cmp_key][cmp_key],
art_cmp_key)
else:
ret[art_cmp_key] = art
return [value for _, value in ret.items()]
def merge_buildinfo(lhs, rhs):
if not lhs or not rhs:
return lhs or rhs
# Check they are compatible
assert lhs["version"] == rhs["version"]
assert lhs["name"] == rhs["name"]
assert lhs["number"] == rhs["number"]
for rhs_module in rhs["modules"]:
lhs_module = find_module(lhs, rhs_module["id"])
lhs_module["artifacts"] = merge_artifacts(lhs_module, rhs_module, key="artifacts",
cmp_key="name")
lhs_module["dependencies"] = merge_artifacts(lhs_module, rhs_module, key="dependencies",
cmp_key="id")
return lhs
def update_build_info(buildinfo, output_file):
build_info = {}
for it in buildinfo:
with open(it) as json_data:
data = json.load(json_data)
build_info = merge_buildinfo(build_info, data)
save(output_file, json.dumps(build_info, indent=4))