Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Do not show internal error when running out of workers #437

Merged
merged 1 commit into from
Jun 7, 2019
Merged

Do not show internal error when running out of workers #437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/435.bugfix.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
No longer show an internal error when we run out of workers due to crashes.
2 changes: 2 additions & 0 deletions changelog/435.feature.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
When the test session is interrupted due to running out of workers, the reason is shown in the test summary
for easier viewing.
21 changes: 15 additions & 6 deletions src/xdist/dsession.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def __init__(self, config):
self._active_nodes = set()
self._failed_nodes_count = 0
self._max_worker_restart = get_default_max_worker_restart(self.config)

# summary message to print at the end of the session
self._summary_report = None
try:
self.terminal = config.pluginmanager.getplugin("terminalreporter")
except KeyError:
Expand Down Expand Up @@ -197,15 +198,23 @@ def worker_errordown(self, node, error):
)
if maximum_reached:
if self._max_worker_restart == 0:
msg = "Worker restarting disabled"
msg = "worker {} crashed and worker restarting disabled".format(
node.gateway.id
)
else:
msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
self.report_line(msg)
msg = "maximum crashed workers reached: %d" % self._max_worker_restart
self._summary_report = msg
self.report_line("\n" + msg)
self.triggershutdown()
else:
self.report_line("Replacing crashed worker %s" % node.gateway.id)
self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
self._clone_node(node)
self._active_nodes.remove(node)

def pytest_terminal_summary(self, terminalreporter):
if self.config.option.verbose >= 0 and self._summary_report:
terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))

def worker_collectionfinish(self, node, ids):
"""worker has finished test collection.

Expand Down Expand Up @@ -315,7 +324,7 @@ def handle_crashitem(self, nodeid, worker):
# XXX count no of failures and retry N times
runner = self.config.pluginmanager.getplugin("runner")
fspath = nodeid.split("::")[0]
msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
msg = "worker %r crashed while running %r" % (worker.gateway.id, nodeid)
rep = runner.TestReport(
nodeid, (fspath, None, fspath), (), "failed", msg, "???"
)
Expand Down
53 changes: 39 additions & 14 deletions testing/acceptance_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,8 +849,8 @@ def test_b(): pass
res = testdir.runpytest(f, "-n1")
res.stdout.fnmatch_lines(
[
"*Replacing crashed worker*",
"*Worker*crashed while running*",
"replacing crashed worker gw*",
"worker*crashed while running*",
"*1 failed*1 passed*",
]
)
Expand All @@ -868,8 +868,8 @@ def test_d(): pass
res = testdir.runpytest(f, "-n2")
res.stdout.fnmatch_lines(
[
"*Replacing crashed worker*",
"*Worker*crashed while running*",
"replacing crashed worker gw*",
"worker*crashed while running*",
"*1 failed*3 passed*",
]
)
Expand All @@ -885,8 +885,8 @@ def test_b(): pass
res = testdir.runpytest(f, "--dist=each", "--tx=popen")
res.stdout.fnmatch_lines(
[
"*Replacing crashed worker*",
"*Worker*crashed while running*",
"replacing crashed worker gw*",
"worker*crashed while running*",
"*1 failed*1 passed*",
]
)
Expand Down Expand Up @@ -922,14 +922,35 @@ def test_d(): pass
res = testdir.runpytest(f, "-n4", "--max-worker-restart=1")
res.stdout.fnmatch_lines(
[
"*Replacing crashed worker*",
"*Maximum crashed workers reached: 1*",
"*Worker*crashed while running*",
"*Worker*crashed while running*",
"replacing crashed worker*",
"maximum crashed workers reached: 1*",
"worker*crashed while running*",
"worker*crashed while running*",
"*2 failed*2 passed*",
]
)

def test_max_worker_restart_tests_queued(self, testdir):
f = testdir.makepyfile(
"""
import os, pytest
@pytest.mark.parametrize('i', range(10))
def test(i): os._exit(1)
"""
)
res = testdir.runpytest(f, "-n2", "--max-worker-restart=3")
res.stdout.fnmatch_lines(
[
"replacing crashed worker*",
"maximum crashed workers reached: 3*",
"worker*crashed while running*",
"worker*crashed while running*",
"* xdist: maximum crashed workers reached: 3 *",
"* 4 failed in *",
]
)
assert "INTERNALERROR" not in res.stdout.str()

def test_max_worker_restart_die(self, testdir):
f = testdir.makepyfile(
"""
Expand All @@ -939,7 +960,10 @@ def test_max_worker_restart_die(self, testdir):
)
res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
res.stdout.fnmatch_lines(
["*Unexpectedly no active workers*", "*INTERNALERROR*"]
[
"* xdist: worker gw* crashed and worker restarting disabled *",
"* no tests ran in *",
]
)

def test_disable_restart(self, testdir):
Expand All @@ -954,9 +978,10 @@ def test_c(): pass
res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
res.stdout.fnmatch_lines(
[
"*Worker restarting disabled*",
"*Worker*crashed while running*",
"*1 failed*2 passed*",
"worker gw* crashed and worker restarting disabled",
"*worker*crashed while running*",
"* xdist: worker gw* crashed and worker restarting disabled *",
"* 1 failed, 2 passed in *",
]
)

Expand Down