diff --git a/changelog/5034.feature.rst b/changelog/5034.feature.rst new file mode 100644 index 00000000000..6ae2def3f13 --- /dev/null +++ b/changelog/5034.feature.rst @@ -0,0 +1 @@ +Improve reporting with ``--lf`` and ``--ff`` (run-last-failure). diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index ef2039539ac..246b8dfd824 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -157,18 +157,11 @@ def __init__(self, config): self.active = any(config.getoption(key) for key in active_keys) self.lastfailed = config.cache.get("cache/lastfailed", {}) self._previously_failed_count = None - self._no_failures_behavior = self.config.getoption("last_failed_no_failures") + self._report_status = None def pytest_report_collectionfinish(self): if self.active and self.config.getoption("verbose") >= 0: - if not self._previously_failed_count: - return None - noun = "failure" if self._previously_failed_count == 1 else "failures" - suffix = " first" if self.config.getoption("failedfirst") else "" - mode = "rerun previous {count} {noun}{suffix}".format( - count=self._previously_failed_count, suffix=suffix, noun=noun - ) - return "run-last-failure: %s" % mode + return "run-last-failure: %s" % self._report_status def pytest_runtest_logreport(self, report): if (report.when == "call" and report.passed) or report.skipped: @@ -196,18 +189,35 @@ def pytest_collection_modifyitems(self, session, config, items): else: previously_passed.append(item) self._previously_failed_count = len(previously_failed) + if not previously_failed: - # running a subset of all tests with recorded failures outside - # of the set of tests currently executing - return - if self.config.getoption("lf"): - items[:] = previously_failed - config.hook.pytest_deselected(items=previously_passed) + # Running a subset of all tests with recorded failures + # only outside of it. + self._report_status = "%d known failures not in selected tests" % ( + len(self.lastfailed), + ) + else: + if self.config.getoption("lf"): + items[:] = previously_failed + config.hook.pytest_deselected(items=previously_passed) + else: # --failedfirst + items[:] = previously_failed + previously_passed + + noun = ( + "failure" if self._previously_failed_count == 1 else "failures" + ) + suffix = " first" if self.config.getoption("failedfirst") else "" + self._report_status = "rerun previous {count} {noun}{suffix}".format( + count=self._previously_failed_count, suffix=suffix, noun=noun + ) + else: + self._report_status = "no previously failed tests, " + if self.config.getoption("last_failed_no_failures") == "none": + self._report_status += "deselecting all items." + config.hook.pytest_deselected(items=items) + items[:] = [] else: - items[:] = previously_failed + previously_passed - elif self._no_failures_behavior == "none": - config.hook.pytest_deselected(items=items) - items[:] = [] + self._report_status += "not deselecting items." def pytest_sessionfinish(self, session): config = self.config @@ -303,8 +313,7 @@ def pytest_addoption(parser): dest="last_failed_no_failures", choices=("all", "none"), default="all", - help="change the behavior when no test failed in the last run or no " - "information about the last failures was found in the cache", + help="which tests to run with no previously (known) failures.", ) diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index a1b4a86b51b..85603edf43f 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -10,6 +10,7 @@ import py import pytest +from _pytest.main import EXIT_NOTESTSCOLLECTED pytest_plugins = ("pytester",) @@ -251,7 +252,13 @@ def test_3(): result = testdir.runpytest("--lf") result.stdout.fnmatch_lines(["*2 passed*1 desel*"]) result = testdir.runpytest("--lf") - result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) + result.stdout.fnmatch_lines( + [ + "collected 3 items", + "run-last-failure: no previously failed tests, not deselecting items.", + "*1 failed*2 passed*", + ] + ) result = testdir.runpytest("--lf", "--cache-clear") result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) @@ -425,7 +432,13 @@ def test_b2(): ) result = testdir.runpytest(test_a, "--lf") - result.stdout.fnmatch_lines(["collected 2 items", "*2 passed in*"]) + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: 2 known failures not in selected tests", + "*2 passed in*", + ] + ) result = testdir.runpytest(test_b, "--lf") result.stdout.fnmatch_lines( @@ -721,7 +734,14 @@ def test_2(): result = testdir.runpytest("--lf", "--lfnf", "all") result.stdout.fnmatch_lines(["*2 passed*"]) result = testdir.runpytest("--lf", "--lfnf", "none") - result.stdout.fnmatch_lines(["*2 desel*"]) + result.stdout.fnmatch_lines( + [ + "collected 2 items / 2 deselected", + "run-last-failure: no previously failed tests, deselecting all items.", + "* 2 deselected in *", + ] + ) + assert result.ret == EXIT_NOTESTSCOLLECTED def test_lastfailed_no_failures_behavior_empty_cache(self, testdir): testdir.makepyfile(