diff --git a/changelog/585.feature.rst b/changelog/585.feature.rst new file mode 100644 index 00000000..318b8c07 --- /dev/null +++ b/changelog/585.feature.rst @@ -0,0 +1 @@ +New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftets.py`` to control the number of workers when ``--numprocesses=auto`` is given in the command-line. diff --git a/changelog/585.trivial.rst b/changelog/585.trivial.rst new file mode 100644 index 00000000..645d78d8 --- /dev/null +++ b/changelog/585.trivial.rst @@ -0,0 +1,7 @@ +``psutil`` is no longer a dependency, as it has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic. + +To those interested to continue to use ``psutil`` to detect the number of CPUs, the new ``pytest_xdist_auto_num_cpus`` hook can be used in the root ``conftest.py`` file of the project:: + + def pytest_xdist_auto_num_workers(config): + import psutil + return psutil.cpu_count(logical=False) diff --git a/src/xdist/newhooks.py b/src/xdist/newhooks.py index f389192c..4ac71960 100644 --- a/src/xdist/newhooks.py +++ b/src/xdist/newhooks.py @@ -55,3 +55,13 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult def pytest_xdist_make_scheduler(config, log): """ return a node scheduler implementation """ + + +@pytest.mark.firstresult +def pytest_xdist_auto_num_workers(config): + """ + Return the number of workers to spawn when ``--numprocesses=auto`` is given in the + command-line. + + .. versionadded:: 2.1 + """ diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 5db39d85..b7f50408 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -5,7 +5,7 @@ import pytest -def auto_detect_cpus(): +def pytest_xdist_auto_num_workers(): try: from os import sched_getaffinity except ImportError: @@ -28,13 +28,9 @@ def cpu_count(): return n if n else 1 -class AutoInt(int): - """Mark value as auto-detected.""" - - def parse_numprocesses(s): if s == "auto": - return AutoInt(auto_detect_cpus()) + return "auto" elif s is not None: return int(s) @@ -187,12 +183,13 @@ def pytest_configure(config): @pytest.mark.tryfirst def pytest_cmdline_main(config): usepdb = config.getoption("usepdb", False) # a core option - if isinstance(config.option.numprocesses, AutoInt): + if config.option.numprocesses == "auto": if usepdb: config.option.numprocesses = 0 config.option.dist = "no" else: - config.option.numprocesses = int(config.option.numprocesses) + auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config) + config.option.numprocesses = auto_num_cpus if config.option.numprocesses: if config.option.dist == "no": diff --git a/testing/test_plugin.py b/testing/test_plugin.py index b8752087..437a05cf 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -51,6 +51,7 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("numprocesses") == 2 config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 99 config = testdir.parseconfigure("-nauto", "--pdb") @@ -62,9 +63,24 @@ def test_auto_detect_cpus(testdir, monkeypatch): monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.setenv("TRAVIS", "true") config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 2 +def test_hook_auto_num_workers(testdir, monkeypatch): + from xdist.plugin import pytest_cmdline_main as check_options + + testdir.makeconftest( + """ + def pytest_xdist_auto_num_workers(): + return 42 + """ + ) + config = testdir.parseconfigure("-nauto") + check_options(config) + assert config.getoption("numprocesses") == 42 + + def test_boxed_with_collect_only(testdir): from xdist.plugin import pytest_cmdline_main as check_options