-
-
Notifications
You must be signed in to change notification settings - Fork 168
/
template.py
473 lines (393 loc) · 15.6 KB
/
template.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
"""Tools related to template management."""
import re
from collections import ChainMap
from contextlib import suppress
from pathlib import Path
from typing import List, Mapping, Optional, Sequence, Set, Tuple
from warnings import warn
import dunamai
import yaml
from iteration_utilities import deepflatten
from packaging.specifiers import SpecifierSet
from packaging.version import Version, parse
from plumbum.cmd import git
from plumbum.machines import local
from pydantic.dataclasses import dataclass
from yamlinclude import YamlIncludeConstructor
from .errors import (
InvalidConfigFileError,
MultipleConfigFilesError,
OldTemplateWarning,
UnknownCopierVersionWarning,
UnsupportedVersionError,
)
from .tools import copier_version
from .types import AnyByStrDict, OptStr, StrSeq, VCSTypes
from .vcs import checkout_latest_tag, clone, get_repo
try:
from functools import cached_property
except ImportError:
# HACK https://github.com/python/mypy/issues/1153#issuecomment-558556828
from backports.cached_property import cached_property # type: ignore
from .types import Literal
# Default list of files in the template to exclude from the rendered project
DEFAULT_EXCLUDE: Tuple[str, ...] = (
"copier.yaml",
"copier.yml",
"~*",
"*.py[co]",
"__pycache__",
".git",
".DS_Store",
".svn",
)
DEFAULT_TEMPLATES_SUFFIX = ".jinja"
# TODO Remove usage of this on Copier v7
COPIER_JINJA_BREAK = SpecifierSet("<=6.0.0a5", prereleases=True)
def filter_config(data: AnyByStrDict) -> Tuple[AnyByStrDict, AnyByStrDict]:
"""Separates config and questions data."""
conf_data: AnyByStrDict = {"secret_questions": set()}
questions_data = {}
for k, v in data.items():
if k == "_secret_questions":
conf_data["secret_questions"].update(v)
elif k.startswith("_"):
conf_data[k[1:]] = v
else:
# Transform simplified questions format into complex
if not isinstance(v, dict):
v = {"default": v}
questions_data[k] = v
if v.get("secret"):
conf_data["secret_questions"].add(k)
return conf_data, questions_data
def load_template_config(conf_path: Path, quiet: bool = False) -> AnyByStrDict:
"""Load the `copier.yml` file.
This is like a simple YAML load, but applying all specific quirks needed
for [the `copier.yml` file][the-copieryml-file].
For example, it supports the `!include` tag with glob includes, and
merges multiple sections.
Params:
conf_path: The path to the `copier.yml` file.
quiet: Used to configure the exception.
Raises:
InvalidConfigFileError: When the file is formatted badly.
"""
YamlIncludeConstructor.add_to_loader_class(
loader_class=yaml.FullLoader, base_dir=conf_path.parent
)
try:
with open(conf_path) as f:
flattened_result = deepflatten(
yaml.load_all(f, Loader=yaml.FullLoader),
depth=2,
types=(list,),
)
return ChainMap(*reversed(list(flattened_result)))
except yaml.parser.ParserError as e:
raise InvalidConfigFileError(conf_path, quiet) from e
def verify_copier_version(version_str: str) -> None:
"""Raise an error if the current Copier version is less than the given version.
Args:
version_str:
Minimal copier version for the template.
"""
installed_version = copier_version()
# Disable check when running copier as editable installation
if installed_version == Version("0.0.0"):
warn(
"Cannot check Copier version constraint.",
UnknownCopierVersionWarning,
)
return
parsed_min = Version(version_str)
if installed_version < parsed_min:
raise UnsupportedVersionError(
f"This template requires Copier version >= {version_str}, "
f"while your version of Copier is {installed_version}."
)
if installed_version.major > parsed_min.major:
warn(
f"This template was designed for Copier {version_str}, "
f"but your version of Copier is {installed_version}. "
f"You could find some incompatibilities.",
OldTemplateWarning,
)
@dataclass
class Template:
"""Object that represents a template and its current state.
See [configuring a template][configuring-a-template].
Attributes:
url:
Absolute origin that points to the template.
It can be:
- A local path.
- A Git url. Note: if something fails, prefix the URL with `git+`.
ref:
The tag to checkout in the template.
Only used if `url` points to a VCS-tracked template.
If `None`, then it will checkout the latest tag, sorted by PEP440.
Otherwise it will checkout the reference used here.
Usually it should be a tag, or `None`.
use_prereleases:
When `True`, the template's *latest* release will consider prereleases.
Only used if:
- `url` points to a VCS-tracked template
- `ref` is `None`.
Helpful if you want to test templates before doing a proper release, but you
need some features that require a proper PEP440 version identifier.
"""
url: str
ref: OptStr = None
use_prereleases: bool = False
@cached_property
def _raw_config(self) -> AnyByStrDict:
"""Get template configuration, raw.
It reads [the `copier.yml` file][the-copieryml-file].
"""
conf_paths = [
p
for p in self.local_abspath.glob("copier.*")
if p.is_file() and re.match(r"\.ya?ml", p.suffix, re.I)
]
if len(conf_paths) > 1:
raise MultipleConfigFilesError(conf_paths)
elif len(conf_paths) == 1:
return load_template_config(conf_paths[0])
return {}
@cached_property
def answers_relpath(self) -> Path:
"""Get the answers file relative path, as specified in the template.
If not specified, returns the default `.copier-answers.yml`.
See [answers_file][].
"""
result = Path(self.config_data.get("answers_file", ".copier-answers.yml"))
assert not result.is_absolute()
return result
@cached_property
def commit(self) -> OptStr:
"""If the template is VCS-tracked, get its commit description."""
if self.vcs == "git":
with local.cwd(self.local_abspath):
return git("describe", "--tags", "--always").strip()
@cached_property
def config_data(self) -> AnyByStrDict:
"""Get config from the template.
It reads [the `copier.yml` file][the-copieryml-file] to get its
[settings][available-settings].
"""
result = filter_config(self._raw_config)[0]
with suppress(KeyError):
verify_copier_version(result["min_copier_version"])
return result
@cached_property
def default_answers(self) -> AnyByStrDict:
"""Get default answers for template's questions."""
return {key: value.get("default") for key, value in self.questions_data.items()}
@cached_property
def envops(self) -> Mapping:
"""Get the Jinja configuration specified in the template, or default values.
See [envops][].
"""
result = self.config_data.get("envops", {})
if "keep_trailing_newline" not in result:
# NOTE: we want to keep trailing newlines in templates as this is what a
# user will most likely expects as a default.
# See https://github.com/copier-org/copier/issues/464
result["keep_trailing_newline"] = True
# TODO Copier v7+ will not use any of these altered defaults
old_defaults = {
"autoescape": False,
"block_end_string": "%]",
"block_start_string": "[%",
"comment_end_string": "#]",
"comment_start_string": "[#",
"keep_trailing_newline": True,
"variable_end_string": "]]",
"variable_start_string": "[[",
}
if self.min_copier_version and self.min_copier_version in COPIER_JINJA_BREAK:
warned = False
for key, value in old_defaults.items():
if key not in result:
if not warned:
warn(
"On future releases, Copier will switch to standard Jinja "
"defaults and this template will not work unless updated.",
FutureWarning,
)
warned = True
result[key] = value
return result
@cached_property
def exclude(self) -> Tuple[str, ...]:
"""Get exclusions specified in the template, or default ones.
See [exclude][].
"""
return tuple(self.config_data.get("exclude", DEFAULT_EXCLUDE))
@cached_property
def jinja_extensions(self) -> Tuple[str, ...]:
"""Get Jinja2 extensions specified in the template, or `()`.
See [jinja_extensions][].
"""
return tuple(self.config_data.get("jinja_extensions", ()))
@cached_property
def metadata(self) -> AnyByStrDict:
"""Get template metadata.
This data, if any, should be saved in the answers file to be able to
restore the template to this same state.
"""
result: AnyByStrDict = {"_src_path": self.url}
if self.commit:
result["_commit"] = self.commit
return result
def migration_tasks(
self, stage: Literal["task", "before", "after"], from_template: "Template"
) -> Sequence[Mapping]:
"""Get migration objects that match current version spec.
Versions are compared using PEP 440.
See [migrations][].
Args:
stage: A valid stage name to find tasks for.
from_template: Original template, from which we are migrating.
"""
result: List[dict] = []
if not (self.version and from_template.version):
return result
extra_env = {
"STAGE": stage,
"VERSION_FROM": str(from_template.commit),
"VERSION_TO": str(self.commit),
"VERSION_PEP440_FROM": str(from_template.version),
"VERSION_PEP440_TO": str(self.version),
}
migration: dict
for migration in self._raw_config.get("_migrations", []):
current = parse(migration["version"])
if self.version >= current > from_template.version:
extra_env = dict(
extra_env,
VERSION_CURRENT=migration["version"],
VERSION_PEP440_CURRENT=str(current),
)
result += [
{"task": task, "extra_env": extra_env}
for task in migration.get(stage, [])
]
return result
@cached_property
def min_copier_version(self) -> Optional[Version]:
"""Gets minimal copier version for the template and validates it.
See [min_copier_version][].
"""
try:
return Version(self.config_data["min_copier_version"])
except KeyError:
return None
@cached_property
def questions_data(self) -> AnyByStrDict:
"""Get questions from the template.
See [questions][].
"""
return filter_config(self._raw_config)[1]
@cached_property
def secret_questions(self) -> Set[str]:
"""Get names of secret questions from the template.
These questions shouldn't be saved into the answers file.
"""
result = set(self.config_data.get("secret_questions", {}))
for key, value in self.questions_data.items():
if value.get("secret"):
result.add(key)
return result
@cached_property
def skip_if_exists(self) -> StrSeq:
"""Get skip patterns from the template.
These files will never be rewritten when rendering the template.
See [skip_if_exists][].
"""
return self.config_data.get("skip_if_exists", ())
@cached_property
def subdirectory(self) -> str:
"""Get the subdirectory as specified in the template.
The subdirectory points to the real template code, allowing the
templater to separate it from other template assets, such as docs,
tests, etc.
See [subdirectory][].
"""
return self.config_data.get("subdirectory", "")
@cached_property
def tasks(self) -> Sequence:
"""Get tasks defined in the template.
See [tasks][].
"""
return self.config_data.get("tasks", [])
@cached_property
def templates_suffix(self) -> str:
"""Get the suffix defined for templates.
By default: `.jinja`.
See [templates_suffix][].
"""
result = self.config_data.get("templates_suffix")
if result is None:
# TODO Delete support for .tmpl default in Copier 7
if (
self.min_copier_version
and self.min_copier_version in COPIER_JINJA_BREAK
):
warn(
"In future Copier releases, the default value for template suffix "
"will change from .tmpl to .jinja, and this template will "
"fail unless updated.",
FutureWarning,
)
return ".tmpl"
return DEFAULT_TEMPLATES_SUFFIX
return result
@cached_property
def local_abspath(self) -> Path:
"""Get the absolute path to the template on disk.
This may clone it if `url` points to a
VCS-tracked template.
"""
result = Path(self.url)
if self.vcs == "git":
result = Path(clone(self.url_expanded, self.ref))
if self.ref is None:
checkout_latest_tag(result, self.use_prereleases)
if not result.is_dir():
raise ValueError("Local template must be a directory.")
return result.absolute()
@cached_property
def url_expanded(self) -> str:
"""Get usable URL.
`url` can be specified in shortcut
format, which wouldn't be understood by the underlying VCS system. This
property returns the expanded version, which should work properly.
"""
return get_repo(self.url) or self.url
@cached_property
def version(self) -> Optional[Version]:
"""PEP440-compliant version object."""
if self.vcs != "git" or not self.commit:
return None
try:
with local.cwd(self.local_abspath):
# Leverage dunamai by default; usually it gets best results
return Version(
dunamai.Version.from_git().serialize(style=dunamai.Style.Pep440)
)
except ValueError:
# A fully descripted commit can be easily detected converted into a
# PEP440 version, because it has the format "<tag>-<count>-g<hash>"
if re.match(r"^.+-\d+-g\w+$", self.commit):
base, count, git_hash = self.commit.rsplit("-", 2)
return Version(f"{base}.dev{count}+{git_hash}")
# If we get here, the commit string is a tag, so we can safely expect
# it's a valid PEP440 version
return Version(self.commit)
@cached_property
def vcs(self) -> Optional[VCSTypes]:
"""Get VCS system used by the template, if any."""
if get_repo(self.url):
return "git"