From f9ba24050ab20975c1aed922819f27e289c9d318 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 15 Aug 2019 13:30:54 +0100 Subject: [PATCH 1/9] add virtualenv workaround to template --- ci/templates/appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/templates/appveyor.yml b/ci/templates/appveyor.yml index eddf1905..6bfce9ae 100644 --- a/ci/templates/appveyor.yml +++ b/ci/templates/appveyor.yml @@ -22,6 +22,9 @@ install: - echo PyPy installed - pypy --version + # Upgrade virtualenv for e.g. more-itertools to be handled properly. + # Pin it to work around https://github.com/tox-dev/tox/issues/1389. + - C:\Python35\python -m pip install -U virtualenv==16.5.0 - C:\Python35\python -m pip install tox test_script: From 30468fc6e084fd9bb008fe55b446e05dd32cd97f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 15 Aug 2019 12:52:17 +0100 Subject: [PATCH 2/9] update docs to match pytest-xdist master/worker terminology --- README.rst | 4 ++-- docs/xdist.rst | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ba839379..ad3fa3d4 100644 --- a/README.rst +++ b/README.rst @@ -139,9 +139,9 @@ examine it. Limitations =========== -For distributed testing the slaves must have the pytest-cov package installed. This is needed since +For distributed testing the workers must have the pytest-cov package installed. This is needed since the plugin must be registered through setuptools for pytest to start the plugin on the -slave. +worker. For subprocess measurement environment variables must make it from the main process to the subprocess. The python used by the subprocess must have pytest-cov installed. The subprocess must diff --git a/docs/xdist.rst b/docs/xdist.rst index 86a63db4..a78cf28e 100644 --- a/docs/xdist.rst +++ b/docs/xdist.rst @@ -5,9 +5,9 @@ Distributed testing (xdist) "load" mode =========== -Distributed testing with dist mode set to load will report on the combined coverage of all slaves. -The slaves may be spread out over any number of hosts and each slave may be located anywhere on the -file system. Each slave will have its subprocesses measured. +Distributed testing with dist mode set to load will report on the combined coverage of all workers. +The workers may be spread out over any number of hosts and each worker may be located anywhere on the +file system. Each worker will have its subprocesses measured. Running distributed testing with dist mode set to load:: @@ -48,8 +48,8 @@ Shows a terminal report:: "each" mode =========== -Distributed testing with dist mode set to each will report on the combined coverage of all slaves. -Since each slave is running all tests this allows generating a combined coverage report for multiple +Distributed testing with dist mode set to each will report on the combined coverage of all workers. +Since each worker is running all tests this allows generating a combined coverage report for multiple environments. Running distributed testing with dist mode set to each:: From d58e16de5b45946d018d956bd31fd7008891159a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 15 Aug 2019 15:38:13 +0100 Subject: [PATCH 3/9] test with pytest-xdist==0.22.0 special-case one tox env for xdist==0.22.0 --- .travis.yml | 58 +++++++++++++++++++++++++++------------------------- appveyor.yml | 8 ++++---- tox.ini | 10 +++++---- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37a572a9..63c82ee2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,62 +17,64 @@ jobs: - env: TOXENV=docs - stage: tests - env: TOXENV=py27-t310-c45 + env: TOXENV=py27-t310-x27-c45 python: '2.7' - - env: TOXENV=py27-t43-c45 + - env: TOXENV=py27-t43-x27-c45 python: '2.7' - - env: TOXENV=py27-t44-c45 + - env: TOXENV=py27-t44-x28-c45 python: '2.7' - - env: TOXENV=py27-t45-c45 + - env: TOXENV=py27-t45-x28-c45 python: '2.7' - - env: TOXENV=py34-t310-c45 + - env: TOXENV=py34-t310-x27-c45 python: '3.4' - - env: TOXENV=py34-t43-c45 + - env: TOXENV=py34-t43-x27-c45 python: '3.4' - - env: TOXENV=py34-t44-c45 + - env: TOXENV=py34-t44-x28-c45 python: '3.4' - - env: TOXENV=py34-t45-c45 + - env: TOXENV=py34-t45-x28-c45 python: '3.4' - - env: TOXENV=py35-t310-c45 + - env: TOXENV=py35-t310-x27-c45 python: '3.5' - - env: TOXENV=py35-t43-c45 + - env: TOXENV=py35-t43-x27-c45 python: '3.5' - - env: TOXENV=py35-t44-c45 + - env: TOXENV=py35-t44-x28-c45 python: '3.5' - - env: TOXENV=py35-t45-c45 + - env: TOXENV=py35-t45-x28-c45 python: '3.5' - - env: TOXENV=py36-t310-c45 + - env: TOXENV=py36-t310-x27-c45 python: '3.6' - - env: TOXENV=py36-t43-c45 + - env: TOXENV=py36-t43-x27-c45 python: '3.6' - - env: TOXENV=py36-t44-c45 + - env: TOXENV=py36-t44-x28-c45 python: '3.6' - - env: TOXENV=py36-t45-c45 + - env: TOXENV=py36-t45-x28-c45 python: '3.6' - - env: TOXENV=py37-t310-c45 + - env: TOXENV=py37-t310-x27-c45 python: '3.7' - - env: TOXENV=py37-t43-c45 + - env: TOXENV=py37-t43-x27-c45 python: '3.7' - - env: TOXENV=py37-t44-c45 + - env: TOXENV=py37-t44-x28-c45 python: '3.7' - - env: TOXENV=py37-t45-c45 + - env: TOXENV=py37-t45-x28-c45 python: '3.7' - - env: TOXENV=pypy-t310-c45 + - env: TOXENV=pypy-t310-x27-c45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t43-c45 + - env: TOXENV=pypy-t43-x27-c45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t44-c45 + - env: TOXENV=pypy-t44-x28-c45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t45-c45 + - env: TOXENV=pypy-t45-x28-c45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy3-t310-c45 + - env: TOXENV=pypy3-t310-x27-c45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t43-c45 + - env: TOXENV=pypy3-t43-x27-c45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t44-c45 + - env: TOXENV=pypy3-t44-x28-c45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t45-c45 + - env: TOXENV=pypy3-t45-x28-c45 python: 'pypy3.5-6.0' + - env: TOXENV=py37-t310-x22-c45 + python: '3.7' - stage: examples python: '3.6' diff --git a/appveyor.yml b/appveyor.yml index 836eaf5a..7ee15937 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,10 +6,10 @@ cache: environment: matrix: - TOXENV: check - - TOXENV: 'py27-t310-c45,py27-t43-c45,py27-t44-c45,py27-t45-c45' - - TOXENV: 'py34-t310-c45,py34-t43-c45,py34-t44-c45,py34-t45-c45' - - TOXENV: 'py35-t310-c45,py35-t43-c45,py35-t44-c45,py35-t45-c45' - - TOXENV: 'pypy-t310-c45,pypy-t43-c45,pypy-t44-c45,pypy-t45-c45' + - TOXENV: 'py27-t310-x27-c45,py27-t43-x27-c45,py27-t44-x28-c45,py27-t45-x28-c45' + - TOXENV: 'py34-t310-x27-c45,py34-t43-x27-c45,py34-t44-x28-c45,py34-t45-x28-c45' + - TOXENV: 'py35-t310-x27-c45,py35-t43-x27-c45,py35-t44-x28-c45,py35-t45-x28-c45' + - TOXENV: 'pypy-t310-x27-c45,pypy-t43-x27-c45,pypy-t44-x28-c45,pypy-t45-x28-c45' init: - ps: echo $env:TOXENV diff --git a/tox.ini b/tox.ini index c5a6f93f..ab76fcad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ -; a generative tox configuration, see: https://testrun.org/tox/latest/config.html#generative-envlist +; a generative tox configuration, see: https://tox.readthedocs.io/en/latest/config.html#generative-envlist [tox] envlist = check - {py27,py34,py35,py36,py37,pypy,pypy3}-{t310,t43,t44,t45}-{c45} + {py27,py34,py35,py36,py37,pypy,pypy3}-{t310-x27,t43-x27,t44-x28,t45-x28}-{c45} + py37-t310-x22-c45 docs [testenv] @@ -19,8 +20,9 @@ setenv = t44: _DEP_PYTEST=pytest==4.4.2 t45: _DEP_PYTEST=pytest==4.5.0 - {t310,t40,t41,t43}: _DEP_PYTESTXDIST=pytest-xdist==1.27.0 - {t44,t45}: _DEP_PYTESTXDIST=pytest-xdist==1.28.0 + x22: _DEP_PYTESTXDIST=pytest-xdist==1.22.0 + x27: _DEP_PYTESTXDIST=pytest-xdist==1.27.0 + x28: _DEP_PYTESTXDIST=pytest-xdist==1.28.0 c44: _DEP_COVERAGE=coverage==4.4.2 c45: _DEP_COVERAGE=coverage==4.5.3 From e8353a3ac3e1e958d9af1b83b91fe40c432f987e Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Aug 2019 09:17:57 +0100 Subject: [PATCH 4/9] Update docs/xdist.rst Co-Authored-By: Daniel Hahler --- docs/xdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/xdist.rst b/docs/xdist.rst index a78cf28e..a2da50e9 100644 --- a/docs/xdist.rst +++ b/docs/xdist.rst @@ -5,7 +5,7 @@ Distributed testing (xdist) "load" mode =========== -Distributed testing with dist mode set to load will report on the combined coverage of all workers. +Distributed testing with dist mode set to "load" will report on the combined coverage of all workers. The workers may be spread out over any number of hosts and each worker may be located anywhere on the file system. Each worker will have its subprocesses measured. From b68946d84129a5a4a267829735f725b3fd53923f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Aug 2019 12:43:25 +0100 Subject: [PATCH 5/9] remove report,coveralls,codecov env injection --- ci/templates/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/templates/.travis.yml b/ci/templates/.travis.yml index f31f67ca..60bf9efa 100644 --- a/ci/templates/.travis.yml +++ b/ci/templates/.travis.yml @@ -18,7 +18,7 @@ jobs: - stage: tests {% for env in tox_environments %} {%+ if not loop.first %}- {% else %} {% endif -%} - env: TOXENV={{ env }}{% if 'cover' in env %},report,coveralls,codecov{% endif -%}{{ '' }} + env: TOXENV={{ env }} {% if env.startswith("pypy-") %} python: 'pypy2.7-6.0' {% elif env.startswith("pypy3-") %} From 00637dea6a7cdb575cfb10d20a88c4559362cf79 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Aug 2019 11:17:27 +0100 Subject: [PATCH 6/9] lengthen toxenv factor names --- .travis.yml | 58 ++++++++++++++++++++++++++-------------------------- appveyor.yml | 8 ++++---- tox.ini | 26 +++++++++++------------ 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index 63c82ee2..766e64a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,63 +17,63 @@ jobs: - env: TOXENV=docs - stage: tests - env: TOXENV=py27-t310-x27-c45 + env: TOXENV=py27-pytest310-xdist27-coverage45 python: '2.7' - - env: TOXENV=py27-t43-x27-c45 + - env: TOXENV=py27-pytest43-xdist27-coverage45 python: '2.7' - - env: TOXENV=py27-t44-x28-c45 + - env: TOXENV=py27-pytest44-xdist28-coverage45 python: '2.7' - - env: TOXENV=py27-t45-x28-c45 + - env: TOXENV=py27-pytest45-xdist28-coverage45 python: '2.7' - - env: TOXENV=py34-t310-x27-c45 + - env: TOXENV=py34-pytest310-xdist27-coverage45 python: '3.4' - - env: TOXENV=py34-t43-x27-c45 + - env: TOXENV=py34-pytest43-xdist27-coverage45 python: '3.4' - - env: TOXENV=py34-t44-x28-c45 + - env: TOXENV=py34-pytest44-xdist28-coverage45 python: '3.4' - - env: TOXENV=py34-t45-x28-c45 + - env: TOXENV=py34-pytest45-xdist28-coverage45 python: '3.4' - - env: TOXENV=py35-t310-x27-c45 + - env: TOXENV=py35-pytest310-xdist27-coverage45 python: '3.5' - - env: TOXENV=py35-t43-x27-c45 + - env: TOXENV=py35-pytest43-xdist27-coverage45 python: '3.5' - - env: TOXENV=py35-t44-x28-c45 + - env: TOXENV=py35-pytest44-xdist28-coverage45 python: '3.5' - - env: TOXENV=py35-t45-x28-c45 + - env: TOXENV=py35-pytest45-xdist28-coverage45 python: '3.5' - - env: TOXENV=py36-t310-x27-c45 + - env: TOXENV=py36-pytest310-xdist27-coverage45 python: '3.6' - - env: TOXENV=py36-t43-x27-c45 + - env: TOXENV=py36-pytest43-xdist27-coverage45 python: '3.6' - - env: TOXENV=py36-t44-x28-c45 + - env: TOXENV=py36-pytest44-xdist28-coverage45 python: '3.6' - - env: TOXENV=py36-t45-x28-c45 + - env: TOXENV=py36-pytest45-xdist28-coverage45 python: '3.6' - - env: TOXENV=py37-t310-x27-c45 + - env: TOXENV=py37-pytest310-xdist27-coverage45 python: '3.7' - - env: TOXENV=py37-t43-x27-c45 + - env: TOXENV=py37-pytest43-xdist27-coverage45 python: '3.7' - - env: TOXENV=py37-t44-x28-c45 + - env: TOXENV=py37-pytest44-xdist28-coverage45 python: '3.7' - - env: TOXENV=py37-t45-x28-c45 + - env: TOXENV=py37-pytest45-xdist28-coverage45 python: '3.7' - - env: TOXENV=pypy-t310-x27-c45 + - env: TOXENV=pypy-pytest310-xdist27-coverage45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t43-x27-c45 + - env: TOXENV=pypy-pytest43-xdist27-coverage45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t44-x28-c45 + - env: TOXENV=pypy-pytest44-xdist28-coverage45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy-t45-x28-c45 + - env: TOXENV=pypy-pytest45-xdist28-coverage45 python: 'pypy2.7-6.0' - - env: TOXENV=pypy3-t310-x27-c45 + - env: TOXENV=pypy3-pytest310-xdist27-coverage45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t43-x27-c45 + - env: TOXENV=pypy3-pytest43-xdist27-coverage45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t44-x28-c45 + - env: TOXENV=pypy3-pytest44-xdist28-coverage45 python: 'pypy3.5-6.0' - - env: TOXENV=pypy3-t45-x28-c45 + - env: TOXENV=pypy3-pytest45-xdist28-coverage45 python: 'pypy3.5-6.0' - - env: TOXENV=py37-t310-x22-c45 + - env: TOXENV=py37-pytest310-xdist22-coverage45 python: '3.7' - stage: examples diff --git a/appveyor.yml b/appveyor.yml index 7ee15937..31428d2b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,10 +6,10 @@ cache: environment: matrix: - TOXENV: check - - TOXENV: 'py27-t310-x27-c45,py27-t43-x27-c45,py27-t44-x28-c45,py27-t45-x28-c45' - - TOXENV: 'py34-t310-x27-c45,py34-t43-x27-c45,py34-t44-x28-c45,py34-t45-x28-c45' - - TOXENV: 'py35-t310-x27-c45,py35-t43-x27-c45,py35-t44-x28-c45,py35-t45-x28-c45' - - TOXENV: 'pypy-t310-x27-c45,pypy-t43-x27-c45,pypy-t44-x28-c45,pypy-t45-x28-c45' + - TOXENV: 'py27-pytest310-xdist27-coverage45,py27-pytest43-xdist27-coverage45,py27-pytest44-xdist28-coverage45,py27-pytest45-xdist28-coverage45' + - TOXENV: 'py34-pytest310-xdist27-coverage45,py34-pytest43-xdist27-coverage45,py34-pytest44-xdist28-coverage45,py34-pytest45-xdist28-coverage45' + - TOXENV: 'py35-pytest310-xdist27-coverage45,py35-pytest43-xdist27-coverage45,py35-pytest44-xdist28-coverage45,py35-pytest45-xdist28-coverage45' + - TOXENV: 'pypy-pytest310-xdist27-coverage45,pypy-pytest43-xdist27-coverage45,pypy-pytest44-xdist28-coverage45,pypy-pytest45-xdist28-coverage45' init: - ps: echo $env:TOXENV diff --git a/tox.ini b/tox.ini index ab76fcad..98e52193 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,8 @@ [tox] envlist = check - {py27,py34,py35,py36,py37,pypy,pypy3}-{t310-x27,t43-x27,t44-x28,t45-x28}-{c45} - py37-t310-x22-c45 + py{27,34,35,36,37,py,py3}-{pytest310-xdist27,pytest43-xdist27,pytest44-xdist28,pytest45-xdist28}-{coverage45} + py37-pytest310-xdist22-coverage45 docs [testenv] @@ -13,19 +13,19 @@ setenv = PYTHONUNBUFFERED=yes # Use env vars for (optional) pinning of deps. - t310: _DEP_PYTEST=pytest==3.10.1 - t40: _DEP_PYTEST=pytest==4.0.2 - t41: _DEP_PYTEST=pytest==4.1.1 - t43: _DEP_PYTEST=pytest==4.3.1 - t44: _DEP_PYTEST=pytest==4.4.2 - t45: _DEP_PYTEST=pytest==4.5.0 + pytest310: _DEP_PYTEST=pytest==3.10.1 + pytest40: _DEP_PYTEST=pytest==4.0.2 + pytest41: _DEP_PYTEST=pytest==4.1.1 + pytest43: _DEP_PYTEST=pytest==4.3.1 + pytest44: _DEP_PYTEST=pytest==4.4.2 + pytest45: _DEP_PYTEST=pytest==4.5.0 - x22: _DEP_PYTESTXDIST=pytest-xdist==1.22.0 - x27: _DEP_PYTESTXDIST=pytest-xdist==1.27.0 - x28: _DEP_PYTESTXDIST=pytest-xdist==1.28.0 + xdist22: _DEP_PYTESTXDIST=pytest-xdist==1.22.0 + xdist27: _DEP_PYTESTXDIST=pytest-xdist==1.27.0 + xdist28: _DEP_PYTESTXDIST=pytest-xdist==1.28.0 - c44: _DEP_COVERAGE=coverage==4.4.2 - c45: _DEP_COVERAGE=coverage==4.5.3 + coverage44: _DEP_COVERAGE=coverage==4.4.2 + coverage45: _DEP_COVERAGE=coverage==4.5.3 passenv = * From fdcd4f1390947fd08faf2aafc7bbdf0222e1ba41 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Aug 2019 11:29:48 +0100 Subject: [PATCH 7/9] update code to match pytest-xdist master/worker terminology --- src/pytest_cov/engine.py | 70 ++++++++++++++++++++-------------------- src/pytest_cov/plugin.py | 16 ++++----- tests/test_pytest_cov.py | 22 ++++++------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/pytest_cov/engine.py b/src/pytest_cov/engine.py index afb98c53..41907e28 100644 --- a/src/pytest_cov/engine.py +++ b/src/pytest_cov/engine.py @@ -29,7 +29,7 @@ def __init__(self, cov_source, cov_report, cov_config, cov_append, cov_branch, c self.combining_cov = None self.data_file = None self.node_descs = set() - self.failed_slaves = [] + self.failed_workers = [] self.topdir = os.getcwd() def pause(self): @@ -131,12 +131,12 @@ def summary(self, stream): total = self.cov.xml_report(ignore_errors=True, outfile=self.cov_report['xml']) stream.write('Coverage XML written to file %s\n' % self.cov.config.xml_output) - # Report on any failed slaves. - if self.failed_slaves: - self.sep(stream, '-', 'coverage: failed slaves') - stream.write('The following slaves failed to return coverage data, ' - 'ensure that pytest-cov is installed on these slaves.\n') - for node in self.failed_slaves: + # Report on any failed workers. + if self.failed_workers: + self.sep(stream, '-', 'coverage: failed workers') + stream.write('The following workers failed to return coverage data, ' + 'ensure that pytest-cov is installed on these workers.\n') + for node in self.failed_workers: stream.write('%s\n' % node.gateway.id) return total @@ -205,28 +205,28 @@ def start(self): self.cov.config.paths['source'] = [self.topdir] def configure_node(self, node): - """Slaves need to know if they are collocated and what files have moved.""" + """Workers need to know if they are collocated and what files have moved.""" - node.slaveinput['cov_master_host'] = socket.gethostname() - node.slaveinput['cov_master_topdir'] = self.topdir - node.slaveinput['cov_master_rsync_roots'] = [str(root) for root in node.nodemanager.roots] + node.workerinput['cov_master_host'] = socket.gethostname() + node.workerinput['cov_master_topdir'] = self.topdir + node.workerinput['cov_master_rsync_roots'] = [str(root) for root in node.nodemanager.roots] def testnodedown(self, node, error): - """Collect data file name from slave.""" + """Collect data file name from worker.""" - # If slave doesn't return any data then it is likely that this - # plugin didn't get activated on the slave side. - if not (hasattr(node, 'slaveoutput') and 'cov_slave_node_id' in node.slaveoutput): - self.failed_slaves.append(node) + # If worker doesn't return any data then it is likely that this + # plugin didn't get activated on the worker side. + if not (hasattr(node, 'workeroutput') and 'cov_worker_node_id' in node.workeroutput): + self.failed_workers.append(node) return - # If slave is not collocated then we must save the data file + # If worker is not collocated then we must save the data file # that it returns to us. - if 'cov_slave_data' in node.slaveoutput: + if 'cov_worker_data' in node.workeroutput: data_suffix = '%s.%s.%06d.%s' % ( socket.gethostname(), os.getpid(), random.randint(0, 999999), - node.slaveoutput['cov_slave_node_id'] + node.workeroutput['cov_worker_node_id'] ) cov = coverage.Coverage(source=self.cov_source, @@ -235,14 +235,14 @@ def testnodedown(self, node, error): config_file=self.cov_config) cov.start() data = CoverageData() - data.read_fileobj(StringIO(node.slaveoutput['cov_slave_data'])) + data.read_fileobj(StringIO(node.workeroutput['cov_worker_data'])) cov.data.update(data) cov.stop() cov.save() - path = node.slaveoutput['cov_slave_path'] + path = node.workeroutput['cov_worker_path'] self.cov.config.paths['source'].append(path) - # Record the slave types that contribute to the data file. + # Record the worker types that contribute to the data file. rinfo = node.gateway._rinfo() node_desc = self.get_node_desc(rinfo.platform, rinfo.version_info) self.node_descs.add(node_desc) @@ -259,24 +259,24 @@ def finish(self): self.cov.save() -class DistSlave(CovController): - """Implementation for distributed slaves.""" +class DistWorker(CovController): + """Implementation for distributed workers.""" def start(self): cleanup() # Determine whether we are collocated with master. - self.is_collocated = (socket.gethostname() == self.config.slaveinput['cov_master_host'] and - self.topdir == self.config.slaveinput['cov_master_topdir']) + self.is_collocated = (socket.gethostname() == self.config.workerinput['cov_master_host'] and + self.topdir == self.config.workerinput['cov_master_topdir']) - # If we are not collocated then rewrite master paths to slave paths. + # If we are not collocated then rewrite master paths to worker paths. if not self.is_collocated: - master_topdir = self.config.slaveinput['cov_master_topdir'] - slave_topdir = self.topdir + master_topdir = self.config.workerinput['cov_master_topdir'] + worker_topdir = self.topdir if self.cov_source is not None: - self.cov_source = [source.replace(master_topdir, slave_topdir) + self.cov_source = [source.replace(master_topdir, worker_topdir) for source in self.cov_source] - self.cov_config = self.cov_config.replace(master_topdir, slave_topdir) + self.cov_config = self.cov_config.replace(master_topdir, worker_topdir) # Erase any previous data and start coverage. self.cov = coverage.Coverage(source=self.cov_source, @@ -303,7 +303,7 @@ def finish(self): # If we are collocated then just inform the master of our # data file to indicate that we have finished. - self.config.slaveoutput['cov_slave_node_id'] = self.nodeid + self.config.workeroutput['cov_worker_node_id'] = self.nodeid else: self.cov.combine() self.cov.save() @@ -312,11 +312,11 @@ def finish(self): # it on the master node. # Send all the data to the master over the channel. - self.config.slaveoutput['cov_slave_path'] = self.topdir - self.config.slaveoutput['cov_slave_node_id'] = self.nodeid + self.config.workeroutput['cov_worker_path'] = self.topdir + self.config.workeroutput['cov_worker_node_id'] = self.nodeid buff = StringIO() self.cov.data.write_fileobj(buff) - self.config.slaveoutput['cov_slave_data'] = buff.getvalue() + self.config.workeroutput['cov_worker_data'] = buff.getvalue() def summary(self, stream): """Only the master reports so do nothing.""" diff --git a/src/pytest_cov/plugin.py b/src/pytest_cov/plugin.py index b57b569b..945ae76e 100644 --- a/src/pytest_cov/plugin.py +++ b/src/pytest_cov/plugin.py @@ -104,7 +104,7 @@ class CovPlugin(object): Delegates all work to a particular implementation based on whether this test process is centralised, a distributed master or a - distributed slave. + distributed worker. """ def __init__(self, options, pluginmanager, start=True): @@ -143,7 +143,7 @@ def __init__(self, options, pluginmanager, start=True): elif start: self.start(engine.Central) - # slave is started in pytest hook + # worker is started in pytest hook def start(self, controller_cls, config=None, nodeid=None): @@ -169,8 +169,8 @@ class Config(object): if self.options.cov_fail_under is None and hasattr(cov_config, 'fail_under'): self.options.cov_fail_under = cov_config.fail_under - def _is_slave(self, session): - return hasattr(session.config, 'slaveinput') + def _is_worker(self, session): + return hasattr(session.config, 'workerinput') def pytest_sessionstart(self, session): """At session start determine our implementation and delegate to it.""" @@ -181,10 +181,10 @@ def pytest_sessionstart(self, session): return self.pid = os.getpid() - if self._is_slave(session): - nodeid = session.config.slaveinput.get('slaveid', + if self._is_worker(session): + nodeid = session.config.workerinput.get('workerid', getattr(session, 'nodeid')) - self.start(engine.DistSlave, session.config, nodeid) + self.start(engine.DistWorker, session.config, nodeid) elif not self._started: self.start(engine.Central) @@ -228,7 +228,7 @@ def pytest_runtestloop(self, session): if self.cov_controller is not None: self.cov_controller.finish() - if not self._is_slave(session) and self._should_report(): + if not self._is_worker(session) and self._should_report(): try: self.cov_total = self.cov_controller.summary(self.cov_report) except CoverageException as exc: diff --git a/tests/test_pytest_cov.py b/tests/test_pytest_cov.py index 7afaf5cb..3b04c953 100644 --- a/tests/test_pytest_cov.py +++ b/tests/test_pytest_cov.py @@ -591,7 +591,7 @@ def test_foo(foo): ]) for line in chain(result.stdout.lines, result.stderr.lines): - assert 'The following slaves failed to return coverage data' not in line + assert 'The following workers failed to return coverage data' not in line assert 'INTERNALERROR' not in line assert result.ret == 0 @@ -605,7 +605,7 @@ def test_dist_collocated(testdir, prop): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-slave-restart=0', + '--max-worker-restart=0', script, *prop.args) result.stdout.fnmatch_lines([ @@ -638,7 +638,7 @@ def test_dist_not_collocated(testdir, prop): '--tx=popen//chdir=%s' % dir2, '--rsyncdir=%s' % script.basename, '--rsyncdir=.coveragerc', - '--max-slave-restart=0', '-s', + '--max-worker-restart=0', '-s', script, *prop.args) result.stdout.fnmatch_lines([ @@ -672,7 +672,7 @@ def test_dist_not_collocated_coveragerc_source(testdir, prop): '--tx=popen//chdir=%s' % dir2, '--rsyncdir=%s' % script.basename, '--rsyncdir=.coveragerc', - '--max-slave-restart=0', '-s', + '--max-worker-restart=0', '-s', script, *prop.args) result.stdout.fnmatch_lines([ @@ -784,7 +784,7 @@ def test_dist_subprocess_collocated(testdir): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-slave-restart=0', + '--max-worker-restart=0', parent_script) result.stdout.fnmatch_lines([ @@ -819,7 +819,7 @@ def test_dist_subprocess_not_collocated(testdir, tmpdir): '--rsyncdir=%s' % child_script, '--rsyncdir=%s' % parent_script, '--rsyncdir=.coveragerc', - '--max-slave-restart=0', + '--max-worker-restart=0', parent_script) result.stdout.fnmatch_lines([ @@ -884,11 +884,11 @@ def test_dist_missing_data(testdir): '--cov-report=term-missing', '--dist=load', '--tx=popen//python=%s' % exe, - '--max-slave-restart=0', + '--max-worker-restart=0', script) result.stdout.fnmatch_lines([ - '*- coverage: failed slaves -*' + '*- coverage: failed workers -*' ]) assert result.ret == 0 @@ -1421,7 +1421,7 @@ def test_cover_conftest_dist(testdir): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-slave-restart=0', + '--max-worker-restart=0', script) assert result.ret == 0 result.stdout.fnmatch_lines([CONF_RESULT]) @@ -1510,7 +1510,7 @@ def test_coveragerc_dist(testdir): '--cov=%s' % script.dirpath(), '--cov-report=term-missing', '-n', '2', - '--max-slave-restart=0', + '--max-worker-restart=0', script) assert result.ret == 0 result.stdout.fnmatch_lines( @@ -1706,7 +1706,7 @@ def test_external_data_file_xdist(testdir): result = testdir.runpytest('-v', '--cov=%s' % script.dirpath(), '-n', '1', - '--max-slave-restart=0', + '--max-worker-restart=0', script) assert result.ret == 0 assert glob.glob(str(testdir.tmpdir.join('some/special/place/coverage-data*'))) From 1a382ba9be821e89645324f912cae49fd74938c4 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Aug 2019 12:11:23 +0100 Subject: [PATCH 8/9] restore backwards compatability with pytest-xdist==1.22.0 --- src/pytest_cov/compat.py | 19 +++++++++++++++++++ src/pytest_cov/engine.py | 37 +++++++++++++++++++++---------------- src/pytest_cov/plugin.py | 8 +++++--- tests/test_pytest_cov.py | 21 ++++++++++++--------- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/pytest_cov/compat.py b/src/pytest_cov/compat.py index 5b4a0bfb..13b50de1 100644 --- a/src/pytest_cov/compat.py +++ b/src/pytest_cov/compat.py @@ -29,3 +29,22 @@ def testsfailed(self): @testsfailed.setter def testsfailed(self, value): setattr(self._session, self._attr, value) + + +def _attrgetter(attr): + """ + Return a callable object that fetches attr from its operand. + + Unlike operator.attrgetter, the returned callable supports an extra two + arg form for a default. + """ + def fn(obj, *args): + return getattr(obj, attr, *args) + + return fn + + +worker = 'slave' # for compatability with pytest-xdist<=1.22.0 +workerid = worker + 'id' +workerinput = _attrgetter(worker + 'input') +workeroutput = _attrgetter(worker + 'output') diff --git a/src/pytest_cov/engine.py b/src/pytest_cov/engine.py index 41907e28..befa1c01 100644 --- a/src/pytest_cov/engine.py +++ b/src/pytest_cov/engine.py @@ -9,7 +9,7 @@ from coverage.data import CoverageData from .embed import cleanup -from .compat import StringIO +from .compat import StringIO, workeroutput, workerinput class CovController(object): @@ -207,26 +207,29 @@ def start(self): def configure_node(self, node): """Workers need to know if they are collocated and what files have moved.""" - node.workerinput['cov_master_host'] = socket.gethostname() - node.workerinput['cov_master_topdir'] = self.topdir - node.workerinput['cov_master_rsync_roots'] = [str(root) for root in node.nodemanager.roots] + workerinput(node).update({ + 'cov_master_host': socket.gethostname(), + 'cov_master_topdir': self.topdir, + 'cov_master_rsync_roots': [str(root) for root in node.nodemanager.roots], + }) def testnodedown(self, node, error): """Collect data file name from worker.""" # If worker doesn't return any data then it is likely that this # plugin didn't get activated on the worker side. - if not (hasattr(node, 'workeroutput') and 'cov_worker_node_id' in node.workeroutput): + output = workeroutput(node, {}) + if 'cov_worker_node_id' not in output: self.failed_workers.append(node) return # If worker is not collocated then we must save the data file # that it returns to us. - if 'cov_worker_data' in node.workeroutput: + if 'cov_worker_data' in output: data_suffix = '%s.%s.%06d.%s' % ( socket.gethostname(), os.getpid(), random.randint(0, 999999), - node.workeroutput['cov_worker_node_id'] + output['cov_worker_node_id'] ) cov = coverage.Coverage(source=self.cov_source, @@ -235,11 +238,11 @@ def testnodedown(self, node, error): config_file=self.cov_config) cov.start() data = CoverageData() - data.read_fileobj(StringIO(node.workeroutput['cov_worker_data'])) + data.read_fileobj(StringIO(output['cov_worker_data'])) cov.data.update(data) cov.stop() cov.save() - path = node.workeroutput['cov_worker_path'] + path = output['cov_worker_path'] self.cov.config.paths['source'].append(path) # Record the worker types that contribute to the data file. @@ -266,12 +269,12 @@ def start(self): cleanup() # Determine whether we are collocated with master. - self.is_collocated = (socket.gethostname() == self.config.workerinput['cov_master_host'] and - self.topdir == self.config.workerinput['cov_master_topdir']) + self.is_collocated = (socket.gethostname() == workerinput(self.config)['cov_master_host'] and + self.topdir == workerinput(self.config)['cov_master_topdir']) # If we are not collocated then rewrite master paths to worker paths. if not self.is_collocated: - master_topdir = self.config.workerinput['cov_master_topdir'] + master_topdir = workerinput(self.config)['cov_master_topdir'] worker_topdir = self.topdir if self.cov_source is not None: self.cov_source = [source.replace(master_topdir, worker_topdir) @@ -303,7 +306,7 @@ def finish(self): # If we are collocated then just inform the master of our # data file to indicate that we have finished. - self.config.workeroutput['cov_worker_node_id'] = self.nodeid + workeroutput(self.config)['cov_worker_node_id'] = self.nodeid else: self.cov.combine() self.cov.save() @@ -312,11 +315,13 @@ def finish(self): # it on the master node. # Send all the data to the master over the channel. - self.config.workeroutput['cov_worker_path'] = self.topdir - self.config.workeroutput['cov_worker_node_id'] = self.nodeid buff = StringIO() self.cov.data.write_fileobj(buff) - self.config.workeroutput['cov_worker_data'] = buff.getvalue() + workeroutput(self.config).update({ + 'cov_worker_path': self.topdir, + 'cov_worker_node_id': self.nodeid, + 'cov_worker_data': buff.getvalue(), + }) def summary(self, stream): """Only the master reports so do nothing.""" diff --git a/src/pytest_cov/plugin.py b/src/pytest_cov/plugin.py index 945ae76e..ad37e59b 100644 --- a/src/pytest_cov/plugin.py +++ b/src/pytest_cov/plugin.py @@ -170,7 +170,7 @@ class Config(object): self.options.cov_fail_under = cov_config.fail_under def _is_worker(self, session): - return hasattr(session.config, 'workerinput') + return compat.workerinput(session.config, None) is not None def pytest_sessionstart(self, session): """At session start determine our implementation and delegate to it.""" @@ -182,8 +182,10 @@ def pytest_sessionstart(self, session): self.pid = os.getpid() if self._is_worker(session): - nodeid = session.config.workerinput.get('workerid', - getattr(session, 'nodeid')) + nodeid = ( + compat.workerinput(session.config) + .get(compat.workerid, getattr(session, 'nodeid')) + ) self.start(engine.DistWorker, session.config, nodeid) elif not self._started: self.start(engine.Central) diff --git a/tests/test_pytest_cov.py b/tests/test_pytest_cov.py index 3b04c953..f7bec6ca 100644 --- a/tests/test_pytest_cov.py +++ b/tests/test_pytest_cov.py @@ -23,9 +23,12 @@ from io import StringIO import pytest_cov.plugin +from pytest_cov import compat coverage, platform, StrictVersion # required for skipif mark on test_cov_min_from_coveragerc +max_worker_restart_0 = "--max-" + compat.worker + "-restart=0" + SCRIPT = ''' import sys, helper @@ -605,7 +608,7 @@ def test_dist_collocated(testdir, prop): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-worker-restart=0', + max_worker_restart_0, script, *prop.args) result.stdout.fnmatch_lines([ @@ -638,7 +641,7 @@ def test_dist_not_collocated(testdir, prop): '--tx=popen//chdir=%s' % dir2, '--rsyncdir=%s' % script.basename, '--rsyncdir=.coveragerc', - '--max-worker-restart=0', '-s', + max_worker_restart_0, '-s', script, *prop.args) result.stdout.fnmatch_lines([ @@ -672,7 +675,7 @@ def test_dist_not_collocated_coveragerc_source(testdir, prop): '--tx=popen//chdir=%s' % dir2, '--rsyncdir=%s' % script.basename, '--rsyncdir=.coveragerc', - '--max-worker-restart=0', '-s', + max_worker_restart_0, '-s', script, *prop.args) result.stdout.fnmatch_lines([ @@ -784,7 +787,7 @@ def test_dist_subprocess_collocated(testdir): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-worker-restart=0', + max_worker_restart_0, parent_script) result.stdout.fnmatch_lines([ @@ -819,7 +822,7 @@ def test_dist_subprocess_not_collocated(testdir, tmpdir): '--rsyncdir=%s' % child_script, '--rsyncdir=%s' % parent_script, '--rsyncdir=.coveragerc', - '--max-worker-restart=0', + max_worker_restart_0, parent_script) result.stdout.fnmatch_lines([ @@ -884,7 +887,7 @@ def test_dist_missing_data(testdir): '--cov-report=term-missing', '--dist=load', '--tx=popen//python=%s' % exe, - '--max-worker-restart=0', + max_worker_restart_0, script) result.stdout.fnmatch_lines([ @@ -1421,7 +1424,7 @@ def test_cover_conftest_dist(testdir): '--cov-report=term-missing', '--dist=load', '--tx=2*popen', - '--max-worker-restart=0', + max_worker_restart_0, script) assert result.ret == 0 result.stdout.fnmatch_lines([CONF_RESULT]) @@ -1510,7 +1513,7 @@ def test_coveragerc_dist(testdir): '--cov=%s' % script.dirpath(), '--cov-report=term-missing', '-n', '2', - '--max-worker-restart=0', + max_worker_restart_0, script) assert result.ret == 0 result.stdout.fnmatch_lines( @@ -1706,7 +1709,7 @@ def test_external_data_file_xdist(testdir): result = testdir.runpytest('-v', '--cov=%s' % script.dirpath(), '-n', '1', - '--max-worker-restart=0', + max_worker_restart_0, script) assert result.ret == 0 assert glob.glob(str(testdir.tmpdir.join('some/special/place/coverage-data*'))) From 6ca30c6017086c05ef1569848ecd06fcf95fec19 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 27 Aug 2019 14:13:33 +0100 Subject: [PATCH 9/9] Update changelog and authors --- AUTHORS.rst | 1 + CHANGELOG.rst | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 78f18f13..206a88bc 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -29,3 +29,4 @@ Authors * Samuel Giffard - https://github.com/Mulugruntz * Семён Марьясин - https://github.com/MarSoft * Alexander Shadchin - https://github.com/shadchin +* Thomas Grainger - https://graingert.co.uk diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cb902c8c..01b5ace0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +2.7.2.dev0 (unreleased) +----------------------- + +* Match pytest-xdist master/worker terminology. + Contributed in `#321 `_ + 2.7.1 (2019-05-03) ------------------