From d3885f25e37b28fe5a50274a5db9819d5e2ce042 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 18 Nov 2020 18:38:02 +0100 Subject: [PATCH 01/15] Parallelize the test runs via pytest-xdist Resolves #2458 --- pytest.ini | 3 +++ setup.cfg | 1 + 2 files changed, 4 insertions(+) diff --git a/pytest.ini b/pytest.ini index 03fc773cf4..df30b82273 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,9 @@ addopts= --doctest-modules --doctest-glob=pkg_resources/api_tests.txt -r sxX + + # `pytest-xdist`: + -n auto doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 diff --git a/setup.cfg b/setup.cfg index 536ec70fab..bda5ab6f1b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,6 +58,7 @@ testing = paver pip>=19.1 # For proper file:// URLs support. jaraco.envs + pytest-xdist docs = # Keep these in sync with docs/requirements.txt From c9188cbf991dbaac6d94b0dd57d3132760fc9e89 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 02:24:08 +0100 Subject: [PATCH 02/15] Sanitize CWD out of sys.path in xdist mode --- setuptools/tests/test_build_meta.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 6d3a997ee0..b0455a62cc 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -1,8 +1,10 @@ import os import shutil +import sys import tarfile import importlib from concurrent import futures +from contextlib import suppress import pytest @@ -44,6 +46,15 @@ def __init__(self, *args, **kwargs): def __call__(self, name, *args, **kw): """Handles aribrary function invocations on the build backend.""" + with suppress(ValueError): + # NOTE: pytest-xdist tends to inject '' into `sys.path` which + # NOTE: may break certain isolation expectations. To address + # NOTE: this, we remove this entry from there so the import + # NOTE: machinery behaves the same as in the default + # NOTE: sequential mode. + # Ref: https://github.com/pytest-dev/pytest-xdist/issues/376 + sys.path.remove('') + os.chdir(self.cwd) os.environ.update(self.env) mod = importlib.import_module(self.backend_name) From be6abaec7183e43c164be21112d0b57307748e1d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 02:26:30 +0100 Subject: [PATCH 03/15] Clarify `test_build_sdist_relative_path_import` --- setuptools/tests/test_build_meta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index b0455a62cc..63c7c275b3 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -349,6 +349,7 @@ def test_build_sdist_relative_path_import(self, tmpdir_cwd): build_files(self._relative_path_import_files) build_backend = self.get_build_backend() with pytest.raises(ImportError): + with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist("temp") @pytest.mark.parametrize('setup_literal, requirements', [ From 8a7a014b8abebcbec942a12d5c63759ada956802 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 02:27:43 +0100 Subject: [PATCH 04/15] Make `get_build_backend` cwd path customizable --- setuptools/tests/test_build_meta.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 63c7c275b3..a117a9beba 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -136,8 +136,10 @@ def run(): class TestBuildMetaBackend: backend_name = 'setuptools.build_meta' - def get_build_backend(self): - return BuildBackend(cwd='.', backend_name=self.backend_name) + def get_build_backend(self, cwd_path=None): + if cwd_path is None: + cwd_path = '.' + return BuildBackend(cwd=cwd_path, backend_name=self.backend_name) @pytest.fixture(params=defns) def build_backend(self, tmpdir, request): From 08ded165701faff86313674b8ee92730902e7a3c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 02:28:22 +0100 Subject: [PATCH 05/15] Replace `tmpdir_cwd` fixture with `tmp_path` --- setuptools/tests/test_build_meta.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index a117a9beba..3f1ba0465f 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -347,12 +347,11 @@ def test_build_sdist_builds_targz_even_if_zip_indicated(self, tmpdir_cwd): """) } - def test_build_sdist_relative_path_import(self, tmpdir_cwd): - build_files(self._relative_path_import_files) - build_backend = self.get_build_backend() - with pytest.raises(ImportError): + def test_build_sdist_relative_path_import(self, tmp_path): + build_files(self._relative_path_import_files, prefix=str(tmp_path)) + build_backend = self.get_build_backend(cwd_path=tmp_path) with pytest.raises(ImportError, match="^No module named 'hello'$"): - build_backend.build_sdist("temp") + build_backend.build_sdist(tmp_path / "temp") @pytest.mark.parametrize('setup_literal, requirements', [ ("'foo'", ['foo']), From 2e077b73e8a391f460b365382408d70439494d43 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 23:10:35 +0100 Subject: [PATCH 06/15] Make `test_pip_upgrade_from_source` xdist-friendly --- setuptools/tests/fixtures.py | 28 ++++++++++++++++++++++++++++ setuptools/tests/test_virtualenv.py | 4 ++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index e8cb7f5237..0480033c5b 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,8 +1,14 @@ +import pathlib +import shutil + import pytest from . import contexts +SRC_DIR = pathlib.Path(__file__).parents[2] + + @pytest.fixture def user_override(monkeypatch): """ @@ -21,3 +27,25 @@ def user_override(monkeypatch): def tmpdir_cwd(tmpdir): with tmpdir.as_cwd() as orig: yield orig + + +@pytest.fixture +def src_dir(): + """The project source directory available via fixture.""" + return SRC_DIR + + +@pytest.fixture +def tmp_src(src_dir, tmp_path): + """Make a copy of the source dir under `$tmp/src`. + + This fixture is useful whenever it's necessary to run `setup.py` + or `pip install` against the source directory when there's no + control over the number of simultaneous invocations. Such + concurrent runs create and delete directories with the same names + under the target directory and so they influence each other's runs + when they are not being executed sequentially. + """ + tmp_src_path = tmp_path / 'src' + shutil.copytree(src_dir, tmp_src_path) + return tmp_src_path diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index 5a942d84c5..79468b1baa 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -85,7 +85,7 @@ def _get_pip_versions(): @pytest.mark.parametrize('pip_version', _get_pip_versions()) -def test_pip_upgrade_from_source(pip_version, virtualenv): +def test_pip_upgrade_from_source(pip_version, tmp_src, virtualenv): """ Check pip can upgrade setuptools from source. """ @@ -104,7 +104,7 @@ def test_pip_upgrade_from_source(pip_version, virtualenv): virtualenv.run(' && '.join(( 'python setup.py -q sdist -d {dist}', 'python setup.py -q bdist_wheel -d {dist}', - )).format(dist=dist_dir), cd=SOURCE_DIR) + )).format(dist=dist_dir), cd=tmp_src) sdist = glob.glob(os.path.join(dist_dir, '*.zip'))[0] wheel = glob.glob(os.path.join(dist_dir, '*.whl'))[0] # Then update from wheel. From fecbc2d2b439385bfee6ac52fa004a61ccde35f6 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 1 Jan 2021 23:54:14 +0100 Subject: [PATCH 07/15] Isolate src for `test_distutils_adoption` --- setuptools/tests/test_distutils_adoption.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index a53773df8c..0e89921c90 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -21,10 +21,10 @@ def run(self, cmd, *args, **kwargs): @pytest.fixture -def venv(tmpdir): +def venv(tmp_path, tmp_src): env = VirtualEnv() - env.root = path.Path(tmpdir) - env.req = os.getcwd() + env.root = path.Path(tmp_path / 'venv') + env.req = str(tmp_src) return env.create() From 5f3d12316dc4e3c3f638786668cee38ff5a73bd3 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 2 Jan 2021 00:15:33 +0100 Subject: [PATCH 08/15] Use tmp src copy in `test_clean_env_install` --- setuptools/tests/test_virtualenv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setuptools/tests/test_virtualenv.py b/setuptools/tests/test_virtualenv.py index 79468b1baa..d72dcbd0e5 100644 --- a/setuptools/tests/test_virtualenv.py +++ b/setuptools/tests/test_virtualenv.py @@ -43,11 +43,11 @@ def bare_virtualenv(): SOURCE_DIR = os.path.join(os.path.dirname(__file__), '../..') -def test_clean_env_install(bare_virtualenv): +def test_clean_env_install(bare_virtualenv, tmp_src): """ Check setuptools can be installed in a clean environment. """ - bare_virtualenv.run(['python', 'setup.py', 'install'], cd=SOURCE_DIR) + bare_virtualenv.run(['python', 'setup.py', 'install'], cd=tmp_src) def _get_pip_versions(): From 7fe4a4054a92782a434d25d4ff85231537892c7f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 17 Jan 2021 21:42:33 -0500 Subject: [PATCH 09/15] Rely on pytest-enabler to enable pytest-xdist when present and enabled. --- pyproject.toml | 3 +++ pytest.ini | 3 --- setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0bc2a46f4f..4e80bdc1a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ addopts = "--flake8" [pytest.enabler.cov] addopts = "--cov" +[pytest.enabler.xdist] +addopts = "-n auto" + [tool.towncrier] package = "setuptools" package_dir = "setuptools" diff --git a/pytest.ini b/pytest.ini index df30b82273..03fc773cf4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,9 +4,6 @@ addopts= --doctest-modules --doctest-glob=pkg_resources/api_tests.txt -r sxX - - # `pytest-xdist`: - -n auto doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 diff --git a/setup.cfg b/setup.cfg index bda5ab6f1b..36c7daeebc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,7 +47,7 @@ testing = pytest-black >= 0.3.7; python_implementation != "PyPy" pytest-cov pytest-mypy; python_implementation != "PyPy" - pytest-enabler + pytest-enabler >= 1.0.1 # local mock From 3c8e758caa11abe63040058ba136a68d2b620ea3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 17 Jan 2021 21:49:57 -0500 Subject: [PATCH 10/15] Avoid indirection in src_dir --- setuptools/tests/fixtures.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index 0480033c5b..4a990eb950 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -6,9 +6,6 @@ from . import contexts -SRC_DIR = pathlib.Path(__file__).parents[2] - - @pytest.fixture def user_override(monkeypatch): """ @@ -32,7 +29,7 @@ def tmpdir_cwd(tmpdir): @pytest.fixture def src_dir(): """The project source directory available via fixture.""" - return SRC_DIR + return pathlib.Path(__file__).parents[2] @pytest.fixture From 31b0896bba77f21984dfad5a0b82fcd57bda9658 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Jan 2021 10:30:11 -0500 Subject: [PATCH 11/15] Rely on rootdir to determine the source. Avoids coupling with position in the test suite. --- setuptools/tests/fixtures.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index 4a990eb950..d975c0fc5b 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,4 +1,3 @@ -import pathlib import shutil import pytest @@ -27,13 +26,7 @@ def tmpdir_cwd(tmpdir): @pytest.fixture -def src_dir(): - """The project source directory available via fixture.""" - return pathlib.Path(__file__).parents[2] - - -@pytest.fixture -def tmp_src(src_dir, tmp_path): +def tmp_src(request, tmp_path): """Make a copy of the source dir under `$tmp/src`. This fixture is useful whenever it's necessary to run `setup.py` @@ -44,5 +37,5 @@ def tmp_src(src_dir, tmp_path): when they are not being executed sequentially. """ tmp_src_path = tmp_path / 'src' - shutil.copytree(src_dir, tmp_src_path) + shutil.copytree(request.config.rootdir, tmp_src_path) return tmp_src_path From f767b4f59b14aa94d7c085173cb9360ba2d187eb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Jan 2021 10:48:38 -0500 Subject: [PATCH 12/15] Extract workaround for pytest-dev/pytest-xdist#376 as a fixture. Invoke the repair at the session level and only when xdist is present. --- setuptools/tests/fixtures.py | 19 +++++++++++++++++++ setuptools/tests/test_build_meta.py | 11 ----------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/setuptools/tests/fixtures.py b/setuptools/tests/fixtures.py index d975c0fc5b..d74b5f031a 100644 --- a/setuptools/tests/fixtures.py +++ b/setuptools/tests/fixtures.py @@ -1,3 +1,5 @@ +import contextlib +import sys import shutil import pytest @@ -39,3 +41,20 @@ def tmp_src(request, tmp_path): tmp_src_path = tmp_path / 'src' shutil.copytree(request.config.rootdir, tmp_src_path) return tmp_src_path + + +@pytest.fixture(autouse=True, scope="session") +def workaround_xdist_376(request): + """ + Workaround pytest-dev/pytest-xdist#376 + + ``pytest-xdist`` tends to inject '' into ``sys.path``, + which may break certain isolation expectations. + Remove the entry so the import + machinery behaves the same irrespective of xdist. + """ + if not request.config.pluginmanager.has_plugin('xdist'): + return + + with contextlib.suppress(ValueError): + sys.path.remove('') diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 3f1ba0465f..5dee857707 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -1,10 +1,8 @@ import os import shutil -import sys import tarfile import importlib from concurrent import futures -from contextlib import suppress import pytest @@ -46,15 +44,6 @@ def __init__(self, *args, **kwargs): def __call__(self, name, *args, **kw): """Handles aribrary function invocations on the build backend.""" - with suppress(ValueError): - # NOTE: pytest-xdist tends to inject '' into `sys.path` which - # NOTE: may break certain isolation expectations. To address - # NOTE: this, we remove this entry from there so the import - # NOTE: machinery behaves the same as in the default - # NOTE: sequential mode. - # Ref: https://github.com/pytest-dev/pytest-xdist/issues/376 - sys.path.remove('') - os.chdir(self.cwd) os.environ.update(self.env) mod = importlib.import_module(self.backend_name) From daf01571d508234fcff707b1a1156c7496c0c131 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Jan 2021 11:02:06 -0500 Subject: [PATCH 13/15] Simplify get_build_backend to simply allow override of cwd. --- setuptools/tests/test_build_meta.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 5dee857707..5331e2f8af 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -11,7 +11,7 @@ class BuildBackendBase: - def __init__(self, cwd=None, env={}, backend_name='setuptools.build_meta'): + def __init__(self, cwd='.', env={}, backend_name='setuptools.build_meta'): self.cwd = cwd self.env = env self.backend_name = backend_name @@ -125,10 +125,8 @@ def run(): class TestBuildMetaBackend: backend_name = 'setuptools.build_meta' - def get_build_backend(self, cwd_path=None): - if cwd_path is None: - cwd_path = '.' - return BuildBackend(cwd=cwd_path, backend_name=self.backend_name) + def get_build_backend(self, **kwargs): + return BuildBackend(backend_name=self.backend_name, **kwargs) @pytest.fixture(params=defns) def build_backend(self, tmpdir, request): @@ -338,7 +336,7 @@ def test_build_sdist_builds_targz_even_if_zip_indicated(self, tmpdir_cwd): def test_build_sdist_relative_path_import(self, tmp_path): build_files(self._relative_path_import_files, prefix=str(tmp_path)) - build_backend = self.get_build_backend(cwd_path=tmp_path) + build_backend = self.get_build_backend(cwd=tmp_path) with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist(tmp_path / "temp") From 77aefc128699182e5b1271162f0b7c557d81b1d5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Jan 2021 11:22:05 -0500 Subject: [PATCH 14/15] Restore test_build_sdist_relative_path_import to its former simple implementation. --- setuptools/tests/test_build_meta.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 5331e2f8af..e117d8e629 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -125,8 +125,8 @@ def run(): class TestBuildMetaBackend: backend_name = 'setuptools.build_meta' - def get_build_backend(self, **kwargs): - return BuildBackend(backend_name=self.backend_name, **kwargs) + def get_build_backend(self): + return BuildBackend(backend_name=self.backend_name) @pytest.fixture(params=defns) def build_backend(self, tmpdir, request): @@ -334,11 +334,11 @@ def test_build_sdist_builds_targz_even_if_zip_indicated(self, tmpdir_cwd): """) } - def test_build_sdist_relative_path_import(self, tmp_path): - build_files(self._relative_path_import_files, prefix=str(tmp_path)) - build_backend = self.get_build_backend(cwd=tmp_path) + def test_build_sdist_relative_path_import(self, tmpdir_cwd): + build_files(self._relative_path_import_files) + build_backend = self.get_build_backend() with pytest.raises(ImportError, match="^No module named 'hello'$"): - build_backend.build_sdist(tmp_path / "temp") + build_backend.build_sdist("temp") @pytest.mark.parametrize('setup_literal, requirements', [ ("'foo'", ['foo']), From 3b571b08feda091838f55d6d0ec3a72325264051 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Jan 2021 12:08:11 -0500 Subject: [PATCH 15/15] Add changelog. --- changelog.d/2459.change.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2459.change.rst diff --git a/changelog.d/2459.change.rst b/changelog.d/2459.change.rst new file mode 100644 index 0000000000..3b8d11a9ed --- /dev/null +++ b/changelog.d/2459.change.rst @@ -0,0 +1 @@ +Tests now run in parallel via pytest-xdist, completing in about half the time. Special thanks to :user:`webknjaz` for hard work implementing test isolation. To run without parallelization, disable the plugin with ``tox -- -p no:xdist``.