diff --git a/changelog/646.feature.rst b/changelog/646.feature.rst new file mode 100644 index 00000000..210c73ee --- /dev/null +++ b/changelog/646.feature.rst @@ -0,0 +1,3 @@ +Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``. + +This is very useful for test suites which are not CPU-bound. diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 1eba32b8..f7d1ff8e 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -5,13 +5,14 @@ import pytest -def pytest_xdist_auto_num_workers(): +def pytest_xdist_auto_num_workers(config): try: import psutil except ImportError: pass else: - count = psutil.cpu_count(logical=False) or psutil.cpu_count() + use_logical = config.option.numprocesses == "logical" + count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count() if count: return count try: @@ -36,8 +37,8 @@ def cpu_count(): def parse_numprocesses(s): - if s == "auto": - return "auto" + if s in ("auto", "logical"): + return s elif s is not None: return int(s) @@ -51,9 +52,10 @@ def pytest_addoption(parser): metavar="numprocesses", action="store", type=parse_numprocesses, - help="shortcut for '--dist=load --tx=NUM*popen', " - "you can use 'auto' here for auto detection CPUs number on " - "host system and it will be 0 when used with --pdb", + help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt " + "to detect physical CPU count. With 'logical', detect logical CPU " + "count. If physical CPU count cannot be found, falls back to logical " + "count. This will be 0 when used with --pdb.", ) group.addoption( "--maxprocesses", @@ -190,7 +192,7 @@ def pytest_configure(config): @pytest.mark.tryfirst def pytest_cmdline_main(config): usepdb = config.getoption("usepdb", False) # a core option - if config.option.numprocesses == "auto": + if config.option.numprocesses in ("auto", "logical"): if usepdb: config.option.numprocesses = 0 config.option.dist = "no" diff --git a/testing/test_plugin.py b/testing/test_plugin.py index c1aac652..58706767 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -69,6 +69,12 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("numprocesses") == 0 assert config.getoption("dist") == "no" + config = testdir.parseconfigure("-nlogical", "--pdb") + check_options(config) + assert config.getoption("usepdb") + assert config.getoption("numprocesses") == 0 + assert config.getoption("dist") == "no" + monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.setenv("TRAVIS", "true") config = testdir.parseconfigure("-nauto") @@ -81,12 +87,16 @@ def test_auto_detect_cpus_psutil(testdir, monkeypatch): psutil = pytest.importorskip("psutil") - monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42) + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42) config = testdir.parseconfigure("-nauto") check_options(config) assert config.getoption("numprocesses") == 42 + config = testdir.parseconfigure("-nlogical") + check_options(config) + assert config.getoption("numprocesses") == 84 + def test_hook_auto_num_workers(testdir, monkeypatch): from xdist.plugin import pytest_cmdline_main as check_options @@ -101,6 +111,10 @@ def pytest_xdist_auto_num_workers(): check_options(config) assert config.getoption("numprocesses") == 42 + config = testdir.parseconfigure("-nlogical") + check_options(config) + assert config.getoption("numprocesses") == 42 + def test_boxed_with_collect_only(testdir): from xdist.plugin import pytest_cmdline_main as check_options