Skip to content

Commit

Permalink
Add 'logical' CPU count choice for --numprocesses
Browse files Browse the repository at this point in the history
This can significantly speed up runs when testing is not CPU-bound.
  • Loading branch information
kroeschl committed Mar 25, 2021
1 parent 1dc0197 commit 041971c
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 9 deletions.
1 change: 1 addition & 0 deletions changelog/XXX.feature.rst
@@ -0,0 +1 @@
Add explicit 'logical' choice to --numprocesses flag
18 changes: 10 additions & 8 deletions src/xdist/plugin.py
Expand Up @@ -5,13 +5,14 @@
import pytest


def pytest_xdist_auto_num_workers():
def pytest_xdist_auto_num_workers(config):
try:
import psutil
except ImportError:
pass
else:
count = psutil.cpu_count(logical=False) or psutil.cpu_count()
use_logical = config.option.numprocesses == "logical"
count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
if count:
return count
try:
Expand All @@ -36,8 +37,8 @@ def cpu_count():


def parse_numprocesses(s):
if s == "auto":
return "auto"
if s == "auto" or s == "logical":
return s
elif s is not None:
return int(s)

Expand All @@ -51,9 +52,10 @@ def pytest_addoption(parser):
metavar="numprocesses",
action="store",
type=parse_numprocesses,
help="shortcut for '--dist=load --tx=NUM*popen', "
"you can use 'auto' here for auto detection CPUs number on "
"host system and it will be 0 when used with --pdb",
help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
"to detect physical CPU count. With 'logical', detect logical CPU "
"count. If physical CPU count cannot be found, falls back to logical "
"count. This will be 0 when used with --pdb.",
)
group.addoption(
"--maxprocesses",
Expand Down Expand Up @@ -190,7 +192,7 @@ def pytest_configure(config):
@pytest.mark.tryfirst
def pytest_cmdline_main(config):
usepdb = config.getoption("usepdb", False) # a core option
if config.option.numprocesses == "auto":
if config.option.numprocesses == "auto" or config.option.numprocesses == "logical":
if usepdb:
config.option.numprocesses = 0
config.option.dist = "no"
Expand Down
16 changes: 15 additions & 1 deletion testing/test_plugin.py
Expand Up @@ -69,6 +69,12 @@ def test_auto_detect_cpus(testdir, monkeypatch):
assert config.getoption("numprocesses") == 0
assert config.getoption("dist") == "no"

config = testdir.parseconfigure("-nlogical", "--pdb")
check_options(config)
assert config.getoption("usepdb")
assert config.getoption("numprocesses") == 0
assert config.getoption("dist") == "no"

monkeypatch.delattr(os, "sched_getaffinity", raising=False)
monkeypatch.setenv("TRAVIS", "true")
config = testdir.parseconfigure("-nauto")
Expand All @@ -81,12 +87,16 @@ def test_auto_detect_cpus_psutil(testdir, monkeypatch):

psutil = pytest.importorskip("psutil")

monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42)
monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)

config = testdir.parseconfigure("-nauto")
check_options(config)
assert config.getoption("numprocesses") == 42

config = testdir.parseconfigure("-nlogical")
check_options(config)
assert config.getoption("numprocesses") == 84


def test_hook_auto_num_workers(testdir, monkeypatch):
from xdist.plugin import pytest_cmdline_main as check_options
Expand All @@ -101,6 +111,10 @@ def pytest_xdist_auto_num_workers():
check_options(config)
assert config.getoption("numprocesses") == 42

config = testdir.parseconfigure("-nlogical")
check_options(config)
assert config.getoption("numprocesses") == 42


def test_boxed_with_collect_only(testdir):
from xdist.plugin import pytest_cmdline_main as check_options
Expand Down

0 comments on commit 041971c

Please sign in to comment.