From a30949c77c0f193e481639133f27b28400789d51 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 25 Feb 2024 12:39:31 -0800 Subject: [PATCH 1/2] Use execnet main_thread_only execmodel Use the execnet main_thread_only execmodel so that code which expects to run in the main thread will just work. This execmodel has been merged to the execnet master branch via pytest-dev/execnet#243, so this patch should not be merged until there is a released version of execnet supporting the main_thread_only execmodel. Closes: pytest-dev/pytest-xdist#620 --- changelog/620.bugfix | 1 + src/xdist/looponfail.py | 2 +- src/xdist/workermanage.py | 6 +++++- testing/test_remote.py | 2 +- testing/test_workermanage.py | 4 ++-- 5 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog/620.bugfix diff --git a/changelog/620.bugfix b/changelog/620.bugfix new file mode 100644 index 00000000..81b3378d --- /dev/null +++ b/changelog/620.bugfix @@ -0,0 +1 @@ +Use the ``execnet`` new ``main_thread_only`` "execmodel" so that code which expects to only run in the main thread will now work as expected. diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index 7e30e4cf..4826c70a 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -82,7 +82,7 @@ def trace(self, *args: object) -> None: print("RemoteControl:", msg) def initgateway(self) -> execnet.Gateway: - return execnet.makegateway("popen") + return execnet.makegateway("execmodel=main_thread_only//popen") def setup(self) -> None: if hasattr(self, "gateway"): diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index 70d95971..5570e1de 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -56,13 +56,15 @@ def __init__( self.testrunuid = self.config.getoption("testrunuid") if self.testrunuid is None: self.testrunuid = uuid.uuid4().hex - self.group = execnet.Group() + self.group = execnet.Group(execmodel="main_thread_only") if specs is None: specs = self._getxspecs() self.specs: list[execnet.XSpec] = [] for spec in specs: if not isinstance(spec, execnet.XSpec): spec = execnet.XSpec(spec) + if getattr(spec, "execmodel", None) != "main_thread_only": + spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}") if not spec.chdir and not spec.popen: spec.chdir = defaultchdir self.group.allocate_id(spec) @@ -90,6 +92,8 @@ def setup_node( spec: execnet.XSpec, putevent: Callable[[tuple[str, dict[str, Any]]], None], ) -> WorkerController: + if getattr(spec, "execmodel", None) != "main_thread_only": + spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}") gw = self.group.makegateway(spec) self.config.hook.pytest_xdist_newgateway(gateway=gw) self.rsync_roots(gw) diff --git a/testing/test_remote.py b/testing/test_remote.py index cbbf758b..0b0334dc 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -49,7 +49,7 @@ def __init__( def setup(self) -> None: self.pytester.chdir() # import os ; os.environ['EXECNET_DEBUG'] = "2" - self.gateway = execnet.makegateway() + self.gateway = execnet.makegateway("execmodel=main_thread_only//popen") self.config = config = self.pytester.parseconfigure() putevent = self.events.put if self.use_callback else None diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index 08b38851..491d0424 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -83,7 +83,7 @@ def test_popen_makegateway_events( assert len(call.specs) == 2 call = hookrecorder.popcall("pytest_xdist_newgateway") - assert call.gateway.spec == execnet.XSpec("popen") + assert call.gateway.spec == execnet.XSpec("execmodel=main_thread_only//popen") assert call.gateway.id == "gw0" call = hookrecorder.popcall("pytest_xdist_newgateway") assert call.gateway.id == "gw1" @@ -177,7 +177,7 @@ def test_hrsync_filter(self, source: Path, dest: Path) -> None: assert names == {"dir", "file.txt", "somedir"} def test_hrsync_one_host(self, source: Path, dest: Path) -> None: - gw = execnet.makegateway("popen//chdir=%s" % dest) + gw = execnet.makegateway("execmodel=main_thread_only//popen//chdir=%s" % dest) finished = [] rsync = HostRSync(source) rsync.add_target_host(gw, finished=lambda: finished.append(1)) From c82fee10effa6b88b5788e1055376f88a71a5946 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 29 Feb 2024 06:59:14 -0300 Subject: [PATCH 2/2] Create 1027.improvement.rst --- changelog/1027.improvement.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/1027.improvement.rst diff --git a/changelog/1027.improvement.rst b/changelog/1027.improvement.rst new file mode 100644 index 00000000..2e6dca23 --- /dev/null +++ b/changelog/1027.improvement.rst @@ -0,0 +1,3 @@ +``pytest-xdist`` workers now always execute the tests in the main thread. + +Previously some tests might end up executing in a separate thread other than ``main`` in the workers, due to some internal `èxecnet`` details. This can cause problems specially with async frameworks where the event loop is running in the ``main`` thread (for example `#620 `__).