From e469fc7b805cc053b7d300678c2353be9cac268e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 24 Aug 2020 14:50:36 -0300 Subject: [PATCH 1/3] Revert "Merge pull request #560 from utapyngo/logical-cpu-count" This reverts commit 0094b29a148c584f3e97f8ad00029820757cf260, reversing changes made to c6255faad4bde9385f5e5880f488b02206b1f073. --- setup.py | 2 +- src/xdist/plugin.py | 23 +++++++++++++++++++++-- testing/test_plugin.py | 16 ++++++++++++---- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index fcba2987..d8ecc9ac 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=6.0.0", "pytest-forked"] +install_requires = ["execnet>=1.1", "pytest>=6.0.0", "pytest-forked"] with open("README.rst") as f: diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 9244de2a..5db39d85 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -1,12 +1,31 @@ +import os import uuid -import psutil import py import pytest def auto_detect_cpus(): - return psutil.cpu_count(logical=False) or psutil.cpu_count() or 1 + try: + from os import sched_getaffinity + except ImportError: + if os.environ.get("TRAVIS") == "true": + # workaround https://bitbucket.org/pypy/pypy/issues/2375 + return 2 + try: + from os import cpu_count + except ImportError: + from multiprocessing import cpu_count + else: + + def cpu_count(): + return len(sched_getaffinity(0)) + + try: + n = cpu_count() + except NotImplementedError: + return 1 + return n if n else 1 class AutoInt(int): diff --git a/testing/test_plugin.py b/testing/test_plugin.py index c800f445..b8752087 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -35,10 +35,17 @@ def test_dist_options(testdir): def test_auto_detect_cpus(testdir, monkeypatch): - import psutil + import os from xdist.plugin import pytest_cmdline_main as check_options - monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 99) + if hasattr(os, "sched_getaffinity"): + monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99))) + elif hasattr(os, "cpu_count"): + monkeypatch.setattr(os, "cpu_count", lambda: 99) + else: + import multiprocessing + + monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99) config = testdir.parseconfigure("-n2") assert config.getoption("numprocesses") == 2 @@ -52,9 +59,10 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("numprocesses") == 0 assert config.getoption("dist") == "no" - monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None) + monkeypatch.delattr(os, "sched_getaffinity", raising=False) + monkeypatch.setenv("TRAVIS", "true") config = testdir.parseconfigure("-nauto") - assert config.getoption("numprocesses") == 1 + assert config.getoption("numprocesses") == 2 def test_boxed_with_collect_only(testdir): From 607d8288d40bd87196e354e561d3f26236f850f8 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 24 Aug 2020 15:17:43 -0300 Subject: [PATCH 2/3] Add psutil extra and introduce pytest_xdist_auto_num_workers hook This makes using psutil optional and opens up the possibility of customization through the pytest_xdist_auto_num_workers hook, making things like #477 possible. Fix #585 --- .appveyor.yml | 1 + .travis.yml | 2 ++ README.rst | 7 ++++--- changelog/585.feature.rst | 1 + changelog/585.trivial.rst | 3 +++ setup.py | 2 +- src/xdist/newhooks.py | 10 ++++++++++ src/xdist/plugin.py | 30 +++++++++++++++++------------- testing/test_plugin.py | 37 +++++++++++++++++++++++++++++++++++++ tox.ini | 16 ++++++++++++---- 10 files changed, 88 insertions(+), 21 deletions(-) create mode 100644 changelog/585.feature.rst create mode 100644 changelog/585.trivial.rst diff --git a/.appveyor.yml b/.appveyor.yml index 0c9cef71..667bf4ae 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,6 +5,7 @@ environment: - TOXENV: "py37-pytestlatest" - TOXENV: "py38-pytestlatest" - TOXENV: "py38-pytestmaster" + - TOXENV: "py38-psutil" install: - C:\Python38\python -m pip install -U pip setuptools virtualenv diff --git a/.travis.yml b/.travis.yml index be759da1..9f728eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ jobs: env: TOXENV=py39-pytestlatest - python: "3.8" env: TOXENV=py38-pytestmaster + - python: "3.8" + env: TOXENV=py38-psutil - stage: deploy python: '3.8' diff --git a/README.rst b/README.rst index cfc30553..af472225 100644 --- a/README.rst +++ b/README.rst @@ -57,10 +57,11 @@ Install the plugin with:: pip install pytest-xdist -or use the package in develop/in-place mode with -a checkout of the `pytest-xdist repository`_ :: - pip install --editable . +To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra:: + + pip install pytest-xdist[psutil] + .. _parallelization: diff --git a/changelog/585.feature.rst b/changelog/585.feature.rst new file mode 100644 index 00000000..d77abb41 --- /dev/null +++ b/changelog/585.feature.rst @@ -0,0 +1 @@ +New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line. diff --git a/changelog/585.trivial.rst b/changelog/585.trivial.rst new file mode 100644 index 00000000..2452f5e5 --- /dev/null +++ b/changelog/585.trivial.rst @@ -0,0 +1,3 @@ +``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra:: + + pip install pytest-xdist[psutil] diff --git a/setup.py b/setup.py index d8ecc9ac..b6fb0f16 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ platforms=["linux", "osx", "win32"], packages=find_packages(where="src"), package_dir={"": "src"}, - extras_require={"testing": ["filelock"]}, + extras_require={"testing": ["filelock"], "psutil": ["psutil>=3.0"]}, entry_points={ "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"] }, diff --git a/src/xdist/newhooks.py b/src/xdist/newhooks.py index f389192c..4ac71960 100644 --- a/src/xdist/newhooks.py +++ b/src/xdist/newhooks.py @@ -55,3 +55,13 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult def pytest_xdist_make_scheduler(config, log): """ return a node scheduler implementation """ + + +@pytest.mark.firstresult +def pytest_xdist_auto_num_workers(config): + """ + Return the number of workers to spawn when ``--numprocesses=auto`` is given in the + command-line. + + .. versionadded:: 2.1 + """ diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 5db39d85..2d8424d9 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -5,9 +5,21 @@ import pytest -def auto_detect_cpus(): +def pytest_xdist_auto_num_workers(): + try: + import psutil + except ImportError: + pass + else: + count = psutil.cpu_count(logical=False) or psutil.cpu_count() + if count: + return count try: from os import sched_getaffinity + + def cpu_count(): + return len(sched_getaffinity(0)) + except ImportError: if os.environ.get("TRAVIS") == "true": # workaround https://bitbucket.org/pypy/pypy/issues/2375 @@ -16,11 +28,6 @@ def auto_detect_cpus(): from os import cpu_count except ImportError: from multiprocessing import cpu_count - else: - - def cpu_count(): - return len(sched_getaffinity(0)) - try: n = cpu_count() except NotImplementedError: @@ -28,13 +35,9 @@ def cpu_count(): return n if n else 1 -class AutoInt(int): - """Mark value as auto-detected.""" - - def parse_numprocesses(s): if s == "auto": - return AutoInt(auto_detect_cpus()) + return "auto" elif s is not None: return int(s) @@ -187,12 +190,13 @@ def pytest_configure(config): @pytest.mark.tryfirst def pytest_cmdline_main(config): usepdb = config.getoption("usepdb", False) # a core option - if isinstance(config.option.numprocesses, AutoInt): + if config.option.numprocesses == "auto": if usepdb: config.option.numprocesses = 0 config.option.dist = "no" else: - config.option.numprocesses = int(config.option.numprocesses) + auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config) + config.option.numprocesses = auto_num_cpus if config.option.numprocesses: if config.option.dist == "no": diff --git a/testing/test_plugin.py b/testing/test_plugin.py index b8752087..c1aac652 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -1,7 +1,11 @@ +from contextlib import suppress + import py import execnet from xdist.workermanage import NodeManager +import pytest + def test_dist_incompatibility_messages(testdir): result = testdir.runpytest("--pdb", "--looponfail") @@ -38,6 +42,11 @@ def test_auto_detect_cpus(testdir, monkeypatch): import os from xdist.plugin import pytest_cmdline_main as check_options + with suppress(ImportError): + import psutil + + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None) + if hasattr(os, "sched_getaffinity"): monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99))) elif hasattr(os, "cpu_count"): @@ -51,6 +60,7 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("numprocesses") == 2 config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 99 config = testdir.parseconfigure("-nauto", "--pdb") @@ -62,9 +72,36 @@ def test_auto_detect_cpus(testdir, monkeypatch): monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.setenv("TRAVIS", "true") config = testdir.parseconfigure("-nauto") + check_options(config) assert config.getoption("numprocesses") == 2 +def test_auto_detect_cpus_psutil(testdir, monkeypatch): + from xdist.plugin import pytest_cmdline_main as check_options + + psutil = pytest.importorskip("psutil") + + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42) + + config = testdir.parseconfigure("-nauto") + check_options(config) + assert config.getoption("numprocesses") == 42 + + +def test_hook_auto_num_workers(testdir, monkeypatch): + from xdist.plugin import pytest_cmdline_main as check_options + + testdir.makeconftest( + """ + def pytest_xdist_auto_num_workers(): + return 42 + """ + ) + config = testdir.parseconfigure("-nauto") + check_options(config) + assert config.getoption("numprocesses") == 42 + + def test_boxed_with_collect_only(testdir): from xdist.plugin import pytest_cmdline_main as check_options diff --git a/tox.ini b/tox.ini index d532a2fd..3774b08a 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,9 @@ envlist= linting py{35,36,37,38,39}-pytestlatest py38-pytestmaster + py38-psutil [testenv] -passenv = USER USERNAME extras = testing deps = pytestlatest: pytest @@ -13,8 +13,16 @@ deps = commands= pytest {posargs} +[testenv:py38-psutil] +extras = + testing + psutil +deps = pytest +commands = + pytest {posargs:-k psutil} + [testenv:linting] -skipsdist = True +skip_install = True usedevelop = True deps = pre-commit @@ -28,9 +36,9 @@ skipsdist = True usedevelop = True passenv = * deps = - towncrier + towncrier commands = - towncrier --version {posargs} --yes + towncrier --version {posargs} --yes [pytest] addopts = -ra From fe48256f1ae45c3c5b2f8259eaee42af71ae0ebb Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 25 Aug 2020 08:45:45 -0300 Subject: [PATCH 3/3] Remove universal wheel setting: pytest-xdist is Python 3 only --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 36f3f4dc..71037d7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [metadata] license_file = LICENSE