/
plugin.py
363 lines (297 loc) · 11.8 KB
/
plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
from itertools import product
import json
import os
import sys
import threading
from typing import Any, Dict, Iterable, List, NoReturn
import importlib_resources
import pluggy
from tox.action import Action
from tox.config import Config, TestenvConfig, _split_env as split_env
from tox.reporter import verbosity1, verbosity2, warning, error
from tox.venv import VirtualEnv
hookimpl = pluggy.HookimplMarker("tox")
# Using thread local for just in case tox uses multiple threads for execution.
# tox seems to be using multiple processes at this point.
thread_locals = threading.local()
thread_locals.is_grouping_started = {}
@hookimpl
def tox_configure(config):
# type: (Config) -> None
verbosity1("running tox-gh-actions")
if not is_log_grouping_enabled(config):
verbosity2(
"disabling log line grouping on GitHub Actions based on the configuration"
)
if not is_running_on_actions():
verbosity1(
"tox-gh-actions won't override envlist "
"because tox is not running in GitHub Actions"
)
return
elif is_env_specified(config):
verbosity1(
"tox-gh-actions won't override envlist because "
"envlist is explicitly given via TOXENV or -e option"
)
return
verbosity2("original envconfigs: {}".format(list(config.envconfigs.keys())))
verbosity2("original envlist_default: {}".format(config.envlist_default))
verbosity2("original envlist: {}".format(config.envlist))
versions = get_python_version_keys()
verbosity2("Python versions: {}".format(versions))
gh_actions_config = parse_config(config._cfg.sections)
verbosity2("tox-gh-actions config: {}".format(gh_actions_config))
if is_running_on_container():
verbosity2(
"not enabling problem matcher as tox seems to be running on a container"
)
# Trying to add a problem matcher from a container without proper host mount can
# cause an error like the following:
# Unable to process command '::add-matcher::/.../matcher.json' successfully.
elif not gh_actions_config["problem_matcher"]:
verbosity2(
"not enabling problem matcher as it's disabled by the problem_matcher "
"option"
)
else:
verbosity2("enabling problem matcher")
print("::add-matcher::" + get_problem_matcher_file_path())
factors = get_factors(gh_actions_config, versions)
verbosity2("using the following factors to decide envlist: {}".format(factors))
envlist = get_envlist_from_factors(config.envlist, factors)
config.envlist_default = config.envlist = envlist
if len(envlist) == 0:
warning(
"tox-gh-actions couldn't find environments matching the provided factors "
"from envlist. Please use `tox -vv` to get more detailed logs."
)
if gh_actions_config["fail_on_no_env"]:
error("Failing the run because the fail_on_no_env option is enabled.")
abort_tox()
verbosity1("overriding envlist with: {}".format(envlist))
@hookimpl
def tox_testenv_create(venv, action):
# type: (VirtualEnv, Action) -> None
if is_log_grouping_enabled(venv.envconfig.config):
start_grouping_if_necessary(venv)
@hookimpl
def tox_testenv_install_deps(venv, action):
# type: (VirtualEnv, Action) -> None
if is_log_grouping_enabled(venv.envconfig.config):
start_grouping_if_necessary(venv)
@hookimpl
def tox_runtest_pre(venv):
# type: (VirtualEnv) -> None
if is_log_grouping_enabled(venv.envconfig.config):
start_grouping_if_necessary(venv)
@hookimpl
def tox_runtest_post(venv):
# type: (VirtualEnv) -> None
if is_log_grouping_enabled(venv.envconfig.config):
print("::endgroup::")
@hookimpl
def tox_cleanup(session):
gh_actions_config = parse_config(session.config._cfg.sections)
# This hook can be called multiple times especially when using parallel mode
if not is_running_on_actions():
return
if not gh_actions_config["problem_matcher"]:
return
verbosity2("disabling problem matcher")
for owner in get_problem_matcher_owners():
print("::remove-matcher owner={}::".format(owner))
def start_grouping_if_necessary(venv):
# type: (VirtualEnv) -> None
"""Start log line grouping when necessary.
This function can be called multiple times when running a test environment
and it ensures that "::group::" is written only once.
We shouldn't call this from tox_package and tox_get_python_executable hooks
because of the timing issue.
"""
envconfig = venv.envconfig # type: TestenvConfig
envname = envconfig.envname
# Do not enable grouping for an environment used for isolated build
# because we don't have a hook to write "::endgroup::" for this environment.
# TODO Make sure the exact condition when isolated_build_env is not set
if envname == getattr(envconfig.config, "isolated_build_env", ""):
return
if thread_locals.is_grouping_started.get(envname, False):
return
thread_locals.is_grouping_started[envname] = True
message = envname
if envconfig.description:
message += " - " + envconfig.description
print("::group::tox: " + message)
def parse_config(config):
# type: (Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, Any]]
"""Parse gh-actions section in tox.ini"""
main_config = config.get("gh-actions", {})
config_python = parse_dict(main_config.get("python", ""))
config_env = {
name: {k: split_env(v) for k, v in parse_dict(conf).items()}
for name, conf in config.get("gh-actions:env", {}).items()
}
return {
# Example of split_env:
# "py{27,38}" => ["py27", "py38"]
"python": {k: split_env(v) for k, v in config_python.items()},
"fail_on_no_env": parse_bool(main_config.get("fail_on_no_env", "false")),
"problem_matcher": parse_bool(main_config.get("problem_matcher", "true")),
"env": config_env,
}
def get_factors(gh_actions_config, versions):
# type: (Dict[str, Dict[str, Any]], Iterable[str]) -> List[str]
"""Get a list of factors"""
factors = [] # type: List[List[str]]
for version in versions:
if version in gh_actions_config["python"]:
show_deprecation_warning_for_old_style_pypy_config(version)
verbosity2("got factors for Python version: {}".format(version))
factors.append(gh_actions_config["python"][version])
break # Shouldn't check remaining versions
for env, env_config in gh_actions_config.get("env", {}).items():
if env in os.environ:
env_value = os.environ[env]
if env_value in env_config:
factors.append(env_config[env_value])
return [x for x in map(lambda f: "-".join(f), product(*factors)) if x]
def show_deprecation_warning_for_old_style_pypy_config(version):
# type: (str) -> None
if version not in {"pypy2", "pypy3"}:
return
warning(
"""PendingDeprecationWarning
Support of old-style PyPy config keys will be removed in tox-gh-actions v3.
Please use "pypy-2" and "pypy-3" instead of "pypy2" and "pypy3".
Example of tox.ini:
[gh-actions]
python =
pypy-2: pypy2
pypy-3: pypy3
# The followings won't work with tox-gh-actions v3
# pypy2: pypy2
# pypy3: pypy3
"""
)
def get_envlist_from_factors(envlist, factors):
# type: (Iterable[str], Iterable[str]) -> List[str]
"""Filter envlist using factors"""
result = []
for env in envlist:
for factor in factors:
env_facts = env.split("-")
if all(f in env_facts for f in factor.split("-")):
result.append(env)
break
return result
def get_python_version_keys():
# type: () -> List[str]
"""Get Python version in string for getting factors from gh-action's config
Examples:
- CPython 2.7.z => [2.7, 2]
- CPython 3.8.z => [3.8, 3]
- PyPy 2.7 (v7.3.z) => [pypy-2.7, pypy-2, pypy2]
- PyPy 3.6 (v7.3.z) => [pypy-3.6, pypy-3, pypy3]
- Pyston based on Python CPython 3.8.8 (v2.2) => [pyston-3.8, pyston-3]
Support of "pypy2" and "pypy3" is for backward compatibility with
tox-gh-actions v2.2.0 and before.
"""
major_version = str(sys.version_info[0])
major_minor_version = ".".join([str(i) for i in sys.version_info[:2]])
if "PyPy" in sys.version:
return [
"pypy-" + major_minor_version,
"pypy-" + major_version,
"pypy" + major_version,
]
elif hasattr(sys, "pyston_version_info"): # Pyston
return [
"pyston-" + major_minor_version,
"pyston-" + major_version,
]
else:
# Assume this is running on CPython
return [major_minor_version, major_version]
def is_running_on_actions():
# type: () -> bool
"""Returns True when running on GitHub Actions"""
# See the following document on which environ to use for this purpose.
# https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables#default-environment-variables
return os.environ.get("GITHUB_ACTIONS") == "true"
def is_running_on_container():
# type: () -> bool
"""Check whether tox is running on a container or not
Only Linux containers are supported.
"""
cgroup_path = "/proc/1/cgroup"
if os.path.exists(cgroup_path):
with open(cgroup_path) as f:
for line in f:
if "containerd" in line:
return True
return False
def is_log_grouping_enabled(config):
# type: (Config) -> bool
"""Returns True when the plugin should enable log line grouping
This plugin won't enable grouping when both --parallel and --parallel-live are
enabled because log lines from different environments will be mixed.
"""
if not is_running_on_actions():
return False
if config.option.parallel > 1 and config.option.parallel_live:
return False
return True
def is_env_specified(config):
# type: (Config) -> bool
"""Returns True when environments are explicitly given"""
if os.environ.get("TOXENV"):
# When TOXENV is a non-empty string
return True
elif config.option.env is not None:
# When command line argument (-e) is given
return True
return False
def get_problem_matcher_file_path():
# type: () -> str
return str(importlib_resources.files("tox_gh_actions") / "matcher.json")
def get_problem_matcher_owners():
# type: () -> List[str]
with open(get_problem_matcher_file_path()) as f:
matcher = json.load(f)
return [m["owner"] for m in matcher["problemMatcher"]]
def parse_bool(value):
# type: (str) -> bool
"""Parse a boolean value in the tox config."""
clean_value = value.strip().lower()
if clean_value in {"true", "1"}:
return True
elif clean_value in {"false", "0"}:
return False
error("Failed to parse a boolean value in tox-gh-actions config: {}")
abort_tox()
def abort_tox():
# type: () -> NoReturn
raise SystemExit(1)
# The following function was copied from
# https://github.com/tox-dev/tox-travis/blob/0.12/src/tox_travis/utils.py#L11-L32
# which is licensed under MIT LICENSE
# https://github.com/tox-dev/tox-travis/blob/0.12/LICENSE
def parse_dict(value):
# type: (str) -> Dict[str, str]
"""Parse a dict value from the tox config.
.. code-block: ini
[travis]
python =
2.7: py27, docs
3.5: py{35,36}
With this config, the value of ``python`` would be parsed
by this function, and would return::
{
'2.7': 'py27, docs',
'3.5': 'py{35,36}',
}
"""
lines = [line.strip() for line in value.strip().splitlines()]
pairs = [line.split(":", 1) for line in lines if line]
return dict((k.strip(), v.strip()) for k, v in pairs)