From c91476980e90baadee76629709cb284417e70387 Mon Sep 17 00:00:00 2001 From: Saravanan Padmanaban Date: Sun, 8 Jan 2023 18:17:37 +0000 Subject: [PATCH 1/3] Dont update cache from xdist worker --- AUTHORS | 1 + src/_pytest/stepwise.py | 4 +++ testing/test_stepwise.py | 77 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/AUTHORS b/AUTHORS index a4ca9926760..7cf1748e606 100644 --- a/AUTHORS +++ b/AUTHORS @@ -315,6 +315,7 @@ Samuel Searles-Bryant Samuele Pedroni Sanket Duthade Sankt Petersbug +Saravanan Padmanaban Segev Finer Serhii Mozghovyi Seth Junot diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index 84f1a6ce8fe..cd2611692e2 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -48,6 +48,8 @@ def pytest_configure(config: Config) -> None: def pytest_sessionfinish(session: Session) -> None: if not session.config.getoption("stepwise"): assert session.config.cache is not None + if hasattr(session.config, "workerinput"): + return # Clear the list of failing tests if the plugin is not active. session.config.cache.set(STEPWISE_CACHE_DIR, []) @@ -119,4 +121,6 @@ def pytest_report_collectionfinish(self) -> Optional[str]: return None def pytest_sessionfinish(self) -> None: + if hasattr(self.config, "workerinput"): + return self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 20781d42cce..2094abc4e50 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -1,6 +1,10 @@ +from pathlib import Path + import pytest +from _pytest.cacheprovider import Cache from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester +from _pytest.stepwise import STEPWISE_CACHE_DIR @pytest.fixture @@ -278,3 +282,76 @@ def test_three(): def test_sw_skip_help(pytester: Pytester) -> None: result = pytester.runpytest("-h") result.stdout.fnmatch_lines("*Implicitly enables --stepwise.") + + +def test_stepwise_xdist_dont_store_lastfailed(pytester: Pytester) -> None: + pytester.makefile( + ext=".ini", + pytest=f"[pytest]\ncache_dir = {pytester.path}\n", + ) + + pytester.makepyfile( + conftest=""" +import pytest + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config) -> None: + config.workerinput = True +""" + ) + pytester.makepyfile( + test_one=""" +def test_one(): + assert False +""" + ) + result = pytester.runpytest("--stepwise") + assert result.ret == pytest.ExitCode.INTERRUPTED + + stepwise_cache_file = ( + pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR + ) + assert not Path(stepwise_cache_file).exists() + + +def test_disabled_stepwise_xdist_dont_clear_cache(pytester: Pytester) -> None: + pytester.makefile( + ext=".ini", + pytest=f"[pytest]\ncache_dir = {pytester.path}\n", + ) + + stepwise_cache_file = ( + pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR + ) + stepwise_cache_dir = stepwise_cache_file.parent + stepwise_cache_dir.mkdir(exist_ok=True, parents=True) + + stepwise_cache_file_relative = f"{Cache._CACHE_PREFIX_VALUES}/{STEPWISE_CACHE_DIR}" + + expected_value = '"test_one.py::test_one"' + content = {f"{stepwise_cache_file_relative}": expected_value} + + pytester.makefile(ext="", **content) + + pytester.makepyfile( + conftest=""" +import pytest + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config) -> None: + config.workerinput = True +""" + ) + pytester.makepyfile( + test_one=""" +def test_one(): + assert True +""" + ) + result = pytester.runpytest() + assert result.ret == 0 + + assert Path(stepwise_cache_file).exists() + with stepwise_cache_file.open() as file_handle: + observed_value = file_handle.readlines() + assert [expected_value] == observed_value From ba509f0addd85f2014f9768e5183ff3e82d3d009 Mon Sep 17 00:00:00 2001 From: Saravanan Padmanaban Date: Mon, 9 Jan 2023 21:40:05 +0000 Subject: [PATCH 2/3] Changelog updated to document PR #10641 --- changelog/10641.bugfix.rst | 1 + src/_pytest/stepwise.py | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog/10641.bugfix.rst diff --git a/changelog/10641.bugfix.rst b/changelog/10641.bugfix.rst new file mode 100644 index 00000000000..febaf5f4b6c --- /dev/null +++ b/changelog/10641.bugfix.rst @@ -0,0 +1 @@ +Fix a race condition when creating or updating the stepwise plugin's cache, which could occur when multiple xdist worker nodes try to simultaneously update the stepwise plugin's cache. \ No newline at end of file diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index cd2611692e2..74ad9dbd4dd 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -49,6 +49,8 @@ def pytest_sessionfinish(session: Session) -> None: if not session.config.getoption("stepwise"): assert session.config.cache is not None if hasattr(session.config, "workerinput"): + # Do not update cache if this process is a xdist worker to prevent + # race conditions (#10641). return # Clear the list of failing tests if the plugin is not active. session.config.cache.set(STEPWISE_CACHE_DIR, []) @@ -122,5 +124,7 @@ def pytest_report_collectionfinish(self) -> Optional[str]: def pytest_sessionfinish(self) -> None: if hasattr(self.config, "workerinput"): + # Do not update cache if this process is a xdist worker to prevent + # race conditions (#10641). return self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) From feda096e58218e7126a506880bd9850646e203cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 21:41:41 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- changelog/10641.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/10641.bugfix.rst b/changelog/10641.bugfix.rst index febaf5f4b6c..ac293f6ae68 100644 --- a/changelog/10641.bugfix.rst +++ b/changelog/10641.bugfix.rst @@ -1 +1 @@ -Fix a race condition when creating or updating the stepwise plugin's cache, which could occur when multiple xdist worker nodes try to simultaneously update the stepwise plugin's cache. \ No newline at end of file +Fix a race condition when creating or updating the stepwise plugin's cache, which could occur when multiple xdist worker nodes try to simultaneously update the stepwise plugin's cache.