diff --git a/changelog/5547.bugfix.rst b/changelog/5547.bugfix.rst new file mode 100644 index 00000000000..0345bf048fc --- /dev/null +++ b/changelog/5547.bugfix.rst @@ -0,0 +1 @@ +``--step-wise`` now handles ``xfail(strict=True)`` markers properly. diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index eb45554904a..a18a58573bd 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -72,7 +72,7 @@ def pytest_collection_modifyitems(self, session, config, items): def pytest_runtest_logreport(self, report): # Skip this hook if plugin is not active or the test is xfailed. - if not self.active or "xfail" in report.keywords: + if not self.active: return if report.failed: diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 40c86fec3d8..591d67b6ce7 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -165,3 +165,56 @@ def test_stop_on_collection_errors(broken_testdir, broken_first): files.reverse() result = broken_testdir.runpytest("-v", "--strict-markers", "--stepwise", *files) result.stdout.fnmatch_lines("*errors during collection*") + + +def test_xfail_handling(testdir): + """Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode + + (#5547) + """ + contents = """ + import pytest + def test_a(): pass + + @pytest.mark.xfail(strict={strict}) + def test_b(): assert {assert_value} + + def test_c(): pass + def test_d(): pass + """ + testdir.makepyfile(contents.format(assert_value="0", strict="False")) + result = testdir.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_a PASSED *", + "*::test_b XFAIL *", + "*::test_c PASSED *", + "*::test_d PASSED *", + "* 3 passed, 1 xfailed in *", + ] + ) + + testdir.makepyfile(contents.format(assert_value="1", strict="True")) + result = testdir.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_a PASSED *", + "*::test_b FAILED *", + "* Interrupted*", + "* 1 failed, 1 passed in *", + ] + ) + + # because we are writing to the same file, mtime might not be affected enough to + # invalidate the cache, making this next run flaky + testdir.tmpdir.join("__pycache__").remove() + testdir.makepyfile(contents.format(assert_value="0", strict="True")) + result = testdir.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_b XFAIL *", + "*::test_c PASSED *", + "*::test_d PASSED *", + "* 2 passed, 1 deselected, 1 xfailed in *", + ] + )