Merge pull request #267 from pytest-dev/better-examples

Recreate the examples

.travis.yml

@@ -5,74 +5,112 @@ env:
   global:
     - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
     - SEGFAULT_SIGNALS=all
-  matrix:
-    - TOXENV=check
-    - TOXENV=docs
-matrix:
+stages:
+  - lint
+  - examples
+  - tests
+jobs:
   include:
-    - python: '2.7'
+    - stage: lint
+      env:
+        - TOXENV=check
+    - stage: lint
+      env:
+        - TOXENV=docs
+    - stage: tests
+      python: '2.7'
       env:
         - TOXENV=py27-t310-c45
-    - python: '2.7'
+    - stage: tests
+      python: '2.7'
       env:
         - TOXENV=py27-t40-c45
-    - python: '2.7'
+    - stage: tests
+      python: '2.7'
       env:
         - TOXENV=py27-t41-c45
-    - python: '3.4'
+    - stage: tests
+      python: '3.4'
       env:
         - TOXENV=py34-t310-c45
-    - python: '3.4'
+    - stage: tests
+      python: '3.4'
       env:
         - TOXENV=py34-t40-c45
-    - python: '3.4'
+    - stage: tests
+      python: '3.4'
       env:
         - TOXENV=py34-t41-c45
-    - python: '3.5'
+    - stage: tests
+      python: '3.5'
       env:
         - TOXENV=py35-t310-c45
-    - python: '3.5'
+    - stage: tests
+      python: '3.5'
       env:
         - TOXENV=py35-t40-c45
-    - python: '3.5'
+    - stage: tests
+      python: '3.5'
       env:
         - TOXENV=py35-t41-c45
-    - python: '3.6'
+    - stage: tests
+      python: '3.6'
       env:
         - TOXENV=py36-t310-c45
-    - python: '3.6'
+    - stage: tests
+      python: '3.6'
       env:
         - TOXENV=py36-t40-c45
-    - python: '3.6'
+    - stage: tests
+      python: '3.6'
       env:
         - TOXENV=py36-t41-c45
-    - python: '3.7'
+    - stage: tests
+      python: '3.7'
       env:
         - TOXENV=py37-t310-c45
-    - python: '3.7'
+    - stage: tests
+      python: '3.7'
       env:
         - TOXENV=py37-t40-c45
-    - python: '3.7'
+    - stage: tests
+      python: '3.7'
       env:
         - TOXENV=py37-t41-c45
-    - python: 'pypy2.7-6.0'
+    - stage: tests
+      python: 'pypy2.7-6.0'
       env:
         - TOXENV=pypy-t310-c45
-    - python: 'pypy2.7-6.0'
+    - stage: tests
+      python: 'pypy2.7-6.0'
       env:
         - TOXENV=pypy-t40-c45
-    - python: 'pypy2.7-6.0'
+    - stage: tests
+      python: 'pypy2.7-6.0'
       env:
         - TOXENV=pypy-t41-c45
-    - python: 'pypy3.5-6.0'
+    - stage: tests
+      python: 'pypy3.5-6.0'
       env:
         - TOXENV=pypy3-t310-c45
-    - python: 'pypy3.5-6.0'
+    - stage: tests
+      python: 'pypy3.5-6.0'
       env:
         - TOXENV=pypy3-t40-c45
-    - python: 'pypy3.5-6.0'
+    - stage: tests
+      python: 'pypy3.5-6.0'
       env:
         - TOXENV=pypy3-t41-c45
+    - stage: examples
+      python: '3.6'
+      script: cd $TARGET; tox -v
+      env:
+          - TARGET=examples/src-layout
+    - stage: examples
+      python: '3.6'
+      script: cd $TARGET; tox -v
+      env:
+          - TARGET=examples/adhoc-layout
 before_install:
   - python --version
   - uname -a

MANIFEST.in

@@ -1,5 +1,10 @@
 graft docs
-graft example
+graft examples
+prune examples/*/.tox
+prune examples/*/htmlcov
+prune examples/adhoc-layout/*.egg-info
+prune examples/src-layout/src/*.egg-info
+
 graft src
 graft ci
 graft tests
@@ -19,4 +24,4 @@ include README.rst
 
 include tox.ini .travis.yml appveyor.yml
 
-global-exclude *.py[cod] __pycache__ *.so
+global-exclude *.py[cod] __pycache__ *.so .coverage

ci/templates/.travis.yml

@@ -5,22 +5,37 @@ env:
   global:
     - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
     - SEGFAULT_SIGNALS=all
-  matrix:
-    - TOXENV=check
-    - TOXENV=docs
-matrix:
+stages:
+  - lint
+  - examples
+  - tests
+jobs:
   include:
+    - stage: lint
+      env:
+        - TOXENV=check
+    - stage: lint
+      env:
+        - TOXENV=docs
 {%- for env in tox_environments %}{{ '' }}
+    - stage: tests
     {% if env.startswith("pypy-") %}
-    - python: 'pypy2.7-6.0'
+      python: 'pypy2.7-6.0'
     {% elif env.startswith("pypy3-") %}
-    - python: 'pypy3.5-6.0'
+      python: 'pypy3.5-6.0'
     {% else %}
-    - python: '{{ "{0[2]}.{0[3]}".format(env) }}'
+      python: '{{ "{0[2]}.{0[3]}".format(env) }}'
     {% endif %}
       env:
         - TOXENV={{ env }}{% if 'cover' in env %},report,coveralls,codecov{% endif -%}
 {%- endfor %}{{ '' }}
+{%- for example in ['src', 'adhoc'] %}{{ '' }}
+    - stage: examples
+      python: '3.6'
+      script: cd $TARGET; tox -v
+      env:
+          - TARGET=examples/{{ example }}-layout
+{%- endfor %}{{ '' }}
 before_install:
   - python --version
   - uname -a

docs/config.rst

@@ -8,9 +8,9 @@ further control of coverage use a coverage config file.
 For example if tests are contained within the directory tree being measured the tests may be
 excluded if desired by using a .coveragerc file with the omit option set::
 
-    pytest --cov-config .coveragerc
-            --cov=myproj
-            myproj/tests/
+    pytest --cov-config=.coveragerc
+           --cov=myproj
+           myproj/tests/
 
 Where the .coveragerc file contains file globs::
 
@@ -41,3 +41,25 @@ In practical terms this means that if you have your coverage configuration in ``
 that you also use ``--cov-config=tox.ini`` or ``--cov-config=setup.cfg``.
 
 You might not be affected but it's unlikely that you won't ever use ``chdir`` in a test.
+
+Reference
+=========
+
+The complete list of command line options is:
+
+  --cov=PATH            Measure coverage for filesystem path. (multi-allowed)
+  --cov-report=type     Type of report to generate: term, term-missing,
+                        annotate, html, xml (multi-allowed). term, term-
+                        missing may be followed by ":skip-covered". annotate,
+                        html and xml may be followed by ":DEST" where DEST
+                        specifies the output location. Use --cov-report= to
+                        not generate any output.
+  --cov-config=path     Config file for coverage. Default: .coveragerc
+  --no-cov-on-fail      Do not report coverage if test run fails. Default:
+                        False
+  --no-cov              Disable coverage report completely (useful for
+                        debuggers). Default: False
+  --cov-fail-under=MIN  Fail if the total coverage is less than MIN.
+  --cov-append          Do not delete coverage but append to current. Default:
+                        False
+  --cov-branch          Enable branch coverage.

docs/index.rst

@@ -12,6 +12,7 @@ Contents:
     debuggers
     xdist
     subprocess-support
+    tox
     plugins
     markers-fixtures
     changelog

docs/tox.rst

@@ -0,0 +1,73 @@
+===
+Tox
+===
+
+When using `tox <https://tox.readthedocs.io/>`_ you can have ultra-compact configuration - you can have all of it in
+``tox.ini``::
+
+    [tox]
+    envlist = ...
+
+    [tool:pytest]
+    ...
+
+    [coverage:paths]
+    ...
+
+    [coverage:run]
+    ...
+
+    [coverage:report]
+    ..
+
+    [testenv]
+    commands = ...
+
+An usual problem users have is that pytest-cov will erase the previous coverage data by default, thus if you run tox
+with multiple environments you'll get incomplete coverage at the end.
+
+To prevent this problem you need to use ``--cov-append``. It's still recommended to clean the previous coverage data to
+have consistent output. A ``tox.ini`` like this should be enough for sequential runs::
+
+    [tox]
+    envlist = clean,py27,py36,...
+
+    [testenv]
+    commands = pytest --cov --cov-append --cov-report=term-missing ...
+    deps =
+        pytest
+        pytest-cov
+
+    [testenv:clean]
+    deps = coverage
+    skip_install = true
+    commands = coverage erase
+
+For parallel runs we need to set some dependencies and have an extra report env like so::
+
+    [tox]
+    envlist = clean,py27,py36,report
+
+    [testenv]
+    commands = pytest --cov --cov-append --cov-report=term-missing
+    deps =
+        pytest
+        pytest-cov
+    depends =
+        {py27,py36}: clean
+        report: py27,py36
+
+    [testenv:report]
+    deps = coverage
+    skip_install = true
+    commands =
+        coverage report
+        coverage html
+
+    [testenv:clean]
+    deps = coverage
+    skip_install = true
+    commands = coverage erase
+
+Depending on your project layout you might need extra configuration, see the working examples at
+https://github.com/pytest-dev/pytest-cov/tree/master/examples for two common layouts.

examples/README.rst

@@ -0,0 +1,15 @@
+Simple examples with ``tox.ini``
+================================
+
+These examples provide necessary configuration to:
+
+* aggregate coverage from multiple interpreters
+* support tox parallel mode
+* run tests on installed code
+
+The `adhoc` layout is the old and problematic layout where you can mix up the installed code
+with the source code. However, these examples will provide correct configuration even for
+the `adhoc` layout.
+
+The `src` layout configuration is less complicated, have that in mind when picking a layout
+for your project.