From 77e5fdfef9abf1526b51699368c9a79d4da61155 Mon Sep 17 00:00:00 2001 From: Ivan Zaikin Date: Thu, 23 Jul 2020 10:18:43 +0700 Subject: [PATCH 1/4] Use psutil.cpu_count(logical=False) in auto_detect_cpus() --- setup.py | 2 +- src/xdist/plugin.py | 19 ++----------------- testing/test_plugin.py | 19 +++++++------------ 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index 3eac3beb..253f8b45 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -install_requires = ["execnet>=1.1", "pytest>=4.4.0", "pytest-forked", "six"] +install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=4.4.0", "pytest-forked", "six"] with open("README.rst") as f: diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 51651263..fea8b5f3 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -1,28 +1,13 @@ -import os import uuid +import psutil import py import pytest def auto_detect_cpus(): try: - from os import sched_getaffinity - except ImportError: - if os.environ.get("TRAVIS") == "true": - # workaround https://bitbucket.org/pypy/pypy/issues/2375 - return 2 - try: - from os import cpu_count - except ImportError: - from multiprocessing import cpu_count - else: - - def cpu_count(): - return len(sched_getaffinity(0)) - - try: - n = cpu_count() + n = psutil.cpu_count(logical=False) except NotImplementedError: return 1 return n if n else 1 diff --git a/testing/test_plugin.py b/testing/test_plugin.py index ca2cc2c8..419842d9 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -35,17 +35,10 @@ def test_dist_options(testdir): def test_auto_detect_cpus(testdir, monkeypatch): - import os + import psutil from xdist.plugin import pytest_cmdline_main as check_options - if hasattr(os, "sched_getaffinity"): - monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99))) - elif hasattr(os, "cpu_count"): - monkeypatch.setattr(os, "cpu_count", lambda: 99) - else: - import multiprocessing - - monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99) + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 99) config = testdir.parseconfigure("-n2") assert config.getoption("numprocesses") == 2 @@ -58,10 +51,12 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("usepdb") assert config.getoption("numprocesses") == 0 - monkeypatch.delattr(os, "sched_getaffinity", raising=False) - monkeypatch.setenv("TRAVIS", "true") + def cpu_count_not_implemented(logical=True): + raise NotImplementedError + + monkeypatch.setattr(psutil, "cpu_count", cpu_count_not_implemented) config = testdir.parseconfigure("-nauto") - assert config.getoption("numprocesses") == 2 + assert config.getoption("numprocesses") == 1 def test_boxed_with_collect_only(testdir): From 1c85f1817a0acb86aacc61b2bf9e746d874fc61b Mon Sep 17 00:00:00 2001 From: Ivan Zaikin Date: Thu, 23 Jul 2020 10:42:04 +0700 Subject: [PATCH 2/4] Add changelog --- changelog/553.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/553.bugfix.rst diff --git a/changelog/553.bugfix.rst b/changelog/553.bugfix.rst new file mode 100644 index 00000000..ee2be329 --- /dev/null +++ b/changelog/553.bugfix.rst @@ -0,0 +1 @@ +When using ``-n auto``, count the number of physical CPU cores instead of logical ones. From 3312dcde61e193fd261dff8b1be1be873d9a530c Mon Sep 17 00:00:00 2001 From: Ivan Zaikin Date: Thu, 23 Jul 2020 10:42:18 +0700 Subject: [PATCH 3/4] Apply black --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 253f8b45..ad4675cd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,12 @@ from setuptools import setup, find_packages -install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=4.4.0", "pytest-forked", "six"] +install_requires = [ + "execnet>=1.1", + "psutil>=3.0.0", + "pytest>=4.4.0", + "pytest-forked", + "six", +] with open("README.rst") as f: From e528a42ddc2464d2e03bdc9331a1ccc50fa8ce25 Mon Sep 17 00:00:00 2001 From: Ivan Zaikin Date: Tue, 28 Jul 2020 08:55:32 +0700 Subject: [PATCH 4/4] Support OpenBSD and NetBSD --- src/xdist/plugin.py | 6 +----- testing/test_plugin.py | 5 +---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index fea8b5f3..e25b231f 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -6,11 +6,7 @@ def auto_detect_cpus(): - try: - n = psutil.cpu_count(logical=False) - except NotImplementedError: - return 1 - return n if n else 1 + return psutil.cpu_count(logical=False) or psutil.cpu_count() or 1 class AutoInt(int): diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 419842d9..31799e20 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -51,10 +51,7 @@ def test_auto_detect_cpus(testdir, monkeypatch): assert config.getoption("usepdb") assert config.getoption("numprocesses") == 0 - def cpu_count_not_implemented(logical=True): - raise NotImplementedError - - monkeypatch.setattr(psutil, "cpu_count", cpu_count_not_implemented) + monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None) config = testdir.parseconfigure("-nauto") assert config.getoption("numprocesses") == 1