diff --git a/.appveyor.yml b/.appveyor.yml index 0c9cef71..667bf4ae 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,6 +5,7 @@ environment: - TOXENV: "py37-pytestlatest" - TOXENV: "py38-pytestlatest" - TOXENV: "py38-pytestmaster" + - TOXENV: "py38-psutil" install: - C:\Python38\python -m pip install -U pip setuptools virtualenv diff --git a/.travis.yml b/.travis.yml index be759da1..9f728eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ jobs: env: TOXENV=py39-pytestlatest - python: "3.8" env: TOXENV=py38-pytestmaster + - python: "3.8" + env: TOXENV=py38-psutil - stage: deploy python: '3.8' diff --git a/README.rst b/README.rst index cfc30553..af472225 100644 --- a/README.rst +++ b/README.rst @@ -57,10 +57,11 @@ Install the plugin with:: pip install pytest-xdist -or use the package in develop/in-place mode with -a checkout of the `pytest-xdist repository`_ :: - pip install --editable . +To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra:: + + pip install pytest-xdist[psutil] + .. _parallelization: diff --git a/changelog/585.feature.rst b/changelog/585.feature.rst new file mode 100644 index 00000000..d77abb41 --- /dev/null +++ b/changelog/585.feature.rst @@ -0,0 +1 @@ +New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line. diff --git a/changelog/585.trivial.rst b/changelog/585.trivial.rst new file mode 100644 index 00000000..2452f5e5 --- /dev/null +++ b/changelog/585.trivial.rst @@ -0,0 +1,3 @@ +``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra:: + + pip install pytest-xdist[psutil] diff --git a/setup.py b/setup.py index d8ecc9ac..b6fb0f16 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ platforms=["linux", "osx", "win32"], packages=find_packages(where="src"), package_dir={"": "src"}, - extras_require={"testing": ["filelock"]}, + extras_require={"testing": ["filelock"], "psutil": ["psutil>=3.0"]}, entry_points={ "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"] }, diff --git a/src/xdist/newhooks.py b/src/xdist/newhooks.py index f389192c..4ac71960 100644 --- a/src/xdist/newhooks.py +++ b/src/xdist/newhooks.py @@ -55,3 +55,13 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult def pytest_xdist_make_scheduler(config, log): """ return a node scheduler implementation """ + + +@pytest.mark.firstresult +def pytest_xdist_auto_num_workers(config): + """ + Return the number of workers to spawn when ``--numprocesses=auto`` is given in the + command-line. + + .. versionadded:: 2.1 + """ diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 5db39d85..2d8424d9 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -5,9 +5,21 @@ import pytest -def auto_detect_cpus(): +def pytest_xdist_auto_num_workers(): + try: + import psutil + except ImportError: + pass + else: + count = psutil.cpu_count(logical=False) or psutil.cpu_count() + if count: + return count try: from os import sched_getaffinity + + def cpu_count(): + return len(sched_getaffinity(0)) + except ImportError: if os.environ.get("TRAVIS") == "true": # workaround https://bitbucket.org/pypy/pypy/issues/2375 @@ -16,11 +28,6 @@ def auto_detect_cpus(): from os import cpu_count except ImportError: from multiprocessing import cpu_count - else: - - def cpu_count(): - return len(sched_getaffinity(0)) - try: n = cpu_count() except NotImplementedError: @@ -28,13 +35,9 @@ def cpu_count(): return n if n else 1 -class AutoInt(int): - """Mark value as auto-detected.""" - - def parse_numprocesses(s): if s == "auto": - return AutoInt(auto_detect_cpus()) + return "auto" elif s is not None: return int(s) @@ -187,12 +190,13 @@ def pytest_configure(config): @pytest.mark.tryfirst def pytest_cmdline_main(config): usepdb = config.getoption("usepdb", False) # a core option - if isinstance(config.option.numprocesses, AutoInt): + if config.option.numprocesses == "auto": if usepdb: config.option.numprocesses = 0 config.option.dist = "no" else: - config.option.numprocesses = int(config.option.numprocesses) + auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config) + config.option.numprocesses = auto_num_cpus if config.option.numprocesses: if config.option.dist == "no": diff --git a/testing/test_plugin.py b/testing/test_plugin.py index b8752087..c1aac652 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -1,7 +1,11 @@ +from contextlib import suppress + import py import execnet from xdist.workermanage import NodeManager +import pytest + def test_dist_incompatibility_messages(testdir): result = testdir.runpytest("--pdb", "--looponfail") @@ -38,6 +42,11 @@ def test_auto_detect_cpus(testdir, monkeypatch): import os from xdist.plugin import pytest_cmdline_main as check_options + with suppress(ImportError): + import psutil + + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None) + if hasattr(os, "sched_getaffinity"): monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99))) elif hasattr(os, "cpu_count"): @@ -51,6 +60,7 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("numprocesses") == 2 config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 99 config = testdir.parseconfigure("-nauto", "--pdb") @@ -62,9 +72,36 @@ def test_auto_detect_cpus(testdir, monkeypatch): monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.setenv("TRAVIS", "true") config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 2 +def test_auto_detect_cpus_psutil(testdir, monkeypatch): + from xdist.plugin import pytest_cmdline_main as check_options + + psutil = pytest.importorskip("psutil") + + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42) + + config = testdir.parseconfigure("-nauto") + check_options(config) + assert config.getoption("numprocesses") == 42 + + +def test_hook_auto_num_workers(testdir, monkeypatch): + from xdist.plugin import pytest_cmdline_main as check_options + + testdir.makeconftest( + """ + def pytest_xdist_auto_num_workers(): + return 42 + """ + ) + config = testdir.parseconfigure("-nauto") + check_options(config) + assert config.getoption("numprocesses") == 42 + + def test_boxed_with_collect_only(testdir): from xdist.plugin import pytest_cmdline_main as check_options diff --git a/tox.ini b/tox.ini index d532a2fd..3774b08a 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,9 @@ envlist= linting py{35,36,37,38,39}-pytestlatest py38-pytestmaster + py38-psutil [testenv] -passenv = USER USERNAME extras = testing deps = pytestlatest: pytest @@ -13,8 +13,16 @@ deps = commands= pytest {posargs} +[testenv:py38-psutil] +extras = + testing + psutil +deps = pytest +commands = + pytest {posargs:-k psutil} + [testenv:linting] -skipsdist = True +skip_install = True usedevelop = True deps = pre-commit @@ -28,9 +36,9 @@ skipsdist = True usedevelop = True passenv = * deps = - towncrier + towncrier commands = - towncrier --version {posargs} --yes + towncrier --version {posargs} --yes [pytest] addopts = -ra