Add nox to run test and environment

.editorconfig

@@ -8,6 +8,7 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
+# for the makefiles use tabs
 [Makefile]
 indent_style = tab
 

.github/workflows/deploy.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Run release_checklist
       run: |
         admin/release_checklist 5.0
- 
+
   deploy:
     runs-on: ubuntu-18.04
     needs: release-check
@@ -44,20 +44,19 @@ jobs:
       uses: seanmiddleditch/gha-setup-ninja@master
     - name: Install dependencies
       run: |
-        make setup-dev PYTHON=python
+        python -m pip install nox
     - name: Lint with flake8
       run: |
-        make lint PYTHON=python
+        nox --non-interactive --session lint
     - name: Test with pytest
       run: |
-        make test CC=$CC PYTHON=python
+        nox --non-interactive --session "tests_compiler($CC)"
     - name: Generate documentation
       run: |
-        make doc PYTHON=python
+        nox --non-interactive --session doc
     - name: Build
       run: |
-        python setup.py sdist bdist_wheel
-        twine check dist/*
+        nox --non-interactive --session build_wheel
     - name: Upload distribution
       if: ${{ success() }}
       uses: actions/upload-artifact@v2
@@ -70,4 +69,4 @@ jobs:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        twine upload dist/*
+        python -m nox --session upload_wheel

.github/workflows/test.yml

@@ -105,8 +105,6 @@ jobs:
     - name: Setup environment
       run: |
         echo "USE_COVERAGE=${{ ( ( matrix.os == 'windows-2019' ) && ( matrix.python-version == '3.7' ) ) || ( ( matrix.os == 'ubuntu-18.04' ) && ( matrix.gcc == 'gcc-6' ) && ( matrix.python-version == '3.8' ) ) }}" >> $GITHUB_ENV
-        echo "CC=${{ matrix.gcc }}" >> $GITHUB_ENV
-        echo "PYTHON=python" >> $GITHUB_ENV
       shell: bash
     - name: Install msys with GCC (Windows)
       if: ${{ startsWith(matrix.os,'windows-') }}
@@ -130,31 +128,21 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v2
-      with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt', 'doc/requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-          ${{ runner.os }}-
     - name: Install dependencies
       run: |
-        make setup-dev
+        python3 -m pip install nox codecov
     - name: Lint files
       run: |
-        make lint
+        nox --non-interactive --session lint
     - name: Check format of files
       if: ${{ ( matrix.python-version != 'pypy3' ) }}
       # Currently fail of this job is OK.
       continue-on-error: true
       run: |
-        make check-format
+        nox --non-interactive --session black
     - name: Test with pytest
       run: |
-        make test TEST_OPTS="--archive_differences"
+        nox --non-interactive --session "tests_compiler(${{ matrix.gcc }})" -- --archive_differences
     - name: Upload pytest test results
       if: ${{ failure() }}
       uses: actions/upload-artifact@v2
@@ -179,13 +167,16 @@ jobs:
         gcc: [gcc-5, gcc-6, gcc-8, clang-10]
 
     steps:
+    - name: Install dependencies
+      run: |
+        python3 -m pip install nox
     - uses: actions/checkout@v2
     - name: Build Docker
       run: |
-        make docker-qa-build CC=${{ matrix.gcc }}
+        python3 -m nox --non-interactive --session "docker_qa_build_compiler(${{ matrix.gcc }})"
     - name: Run Docker
       run: |
-        make docker-qa TEST_OPTS="--archive_differences" CC=${{ matrix.gcc }}
+        python3 -m nox --non-interactive --session "docker_qa_run_compiler(${{ matrix.gcc }})"
     - name: Upload pytest test results
       if: ${{ failure() }}
       uses: actions/upload-artifact@v2

.gitignore

@@ -34,6 +34,8 @@ gcovr-*/
 
 # VS Code
 .vscode
+*.code-workspace
 
 # PyCharm
 .idea
+

CHANGELOG.rst

@@ -28,6 +28,7 @@ Internal changes:
  - New parser for ``.gcov`` file format, should be more robust. (:issue:`512`)
  - Add option to run all comiler versions at once. (:issue:`514`)
  - Fix globing of reference data in tests. (:issue:`533`)
+ - Replace makefile for starting tests with noxfile.py. (:issue:`516`)
 
 5.0 (11 June 2021)
 ------------------

CONTRIBUTING.rst

@@ -71,7 +71,7 @@ Please follow this checklist for your pull request:
 -   **Does it work?**
     Please run the tests locally::
 
-        make test
+        python3 -m nox
 
     (see also: :ref:`test suite`)
 
@@ -92,13 +92,13 @@ Please follow this checklist for your pull request:
     The source code should conform to the :pep:`8` standard.
     Please check your code::
 
-        make lint
+        python3 -m nox --session lint
 
         # or:
 
-        python3 -m flake8 doc gcovr --ignore E501,W503
+        python3 -m flake8 doc gcovr
 
-    The command ``make qa`` will run the linter, run the tests,
+    The command ``python3 -m nox`` will run the linter, run the tests,
     and check that the docs can be built.
 
 -   **Add yourself as an author.**
@@ -145,8 +145,8 @@ How to set up a development environment
 
 For working on gcovr, you will need a supported version of Python 3,
 GCC version 5, 6 or 8 (other GCC versions are supported by gcovr,
-but will cause spurious test failures), ``make``, ``cmake`` and
-``ninja``.
+but will cause spurious test failures) or clang version 10, ``make``,
+``cmake`` and ``ninja``.
 Please make sure that the tools are in the system ``PATH``.
 On **Windows**, you will need to install a GCC toolchain as the
 tests expect a Unix-like environment. You can use MinGW-W64 or MinGW.
@@ -180,26 +180,15 @@ is needed.
 
 -   (Optional) Set up a virtualenv (e.g. with ``python3 -m venv my-venv``)
 
--   Install gcovr in development mode, and install the test requirements::
-
-        make setup-dev  # install all test + doc dependencies
-
-        # or:
+-   Install gcovr in development mode, and install nox::
 
         pip install -e .
-        pip install -r requirements.txt
+        pip install nox
 
     You can then run gcovr as ``gcovr`` or ``python3 -m gcovr``.
 
     Run the tests to verify that everything works (see :ref:`test suite`).
 
--   (Optional) Install documentation requirements::
-
-        # would be already done by `make setup-dev`
-        pip install -r doc/requirements.txt
-
-    See ``doc/README.txt`` for details on working with the documentation.
-
 -   (Optional) Activate GitHub Actions for your forked repository,
     so that the cross-platform compatibility tests get run
     whenever you push your work to your repository.
@@ -224,6 +213,7 @@ Path                    Description
 ``/gcovr/__main__.py``  command line interface + top-level behaviour
 ``/gcovr/templates/``   HTML report templates
 ``/gcovr/tests/``       unit tests + integration test corpus
+``/noxfile.py``         Definition of tests tasks
 ``/setup.py``           Python package configuration
 ``/doc/``               documentation
 ``/doc/sources/``       user guide + website
@@ -240,28 +230,32 @@ are in ``gcovr.generator.html`` and respective modules.
 Test suite
 ----------
 
-The QA process (``make qa``) consists of multiple parts:
-
-- linting (``make lint``)
+The QA process (``python3 -m nox``) consists of multiple parts:
 
-- checking format (``make check-format``)
+- linting and checking format(``python3 -m nox --session lint``)
 
-- tests (``make test``)
+- tests (``python3 -m nox --session tests``)
 
    - unit tests in ``gcovr/tests``
    - integration tests in ``gcovr/tests``
    - documentation examples in ``doc/examples``
 
-- documentation build (``make doc``)
+- documentation build (``python3 -m nox --session doc``)
 
 The tests are in the ``gcovr/tests`` directory.
-You can run the tests with ``make test`` or ``python3 -m pytest -- gcovr doc/examples``.
+You can run the tests with ``python3 -m nox --session tests``
+for the default GCC version (used from environment ``CC``iv availabele, else gcc-5.
+You can also select the gcc version if you run the tests with e.g.
+``python3 -m nox --session tests_compiler(gcc-8)``.
 
 There are unit tests for some parts of gcovr,
 and a comprehensive corpus of example projects
 that are executed as the ``test_gcovr.py`` integration test.
 Each ``gcovr/tests/*`` directory is one such example project.
 
+You can format files with ``python3 -m nox --session black FileToFormat``)
+``
+
 The next sections discuss
 the :ref:`structure of integration tests <integration tests>`,
 how to :ref:`run and filter tests <run tests>`,
@@ -308,53 +302,53 @@ Each Makefile contains the following targets:
 Run and filter tests
 ~~~~~~~~~~~~~~~~~~~~
 
-To run all tests, use ``make test`` or ``make qa``.
+To run all tests, use ``python3 -m nox``.
 The tests currently assume that you are using GCC 5
 and have set up a :ref:`development environment <development environment>`.
-You can select a different GCC version by setting the CC argument.
+You can select a different GCC version by setting the CC environment variable.
 Supported versions are ``CC=gcc-5``, ``CC=gcc-6``, ``CC=gcc-8`` and ``clang-10``.
 
-You can run the tests with additional options by setting ``TEST_OPTS`` variable.
-Run all tests after each change is a bit slow, therefore you can limit the tests
-to a specific test file, example project, or output format.
+You can run the tests with additional options by adding ``--`` and then the options
+to the test invocation. Run all tests after each change is a bit slow, therefore you can
+limit the tests to a specific test file, example project, or output format.
 For example:
 
 .. code:: bash
 
     # run only XML tests
-    make test TEST_OPTS="-k 'xml'"
+    python3 -m nox --session tests -- -k 'xml'
 
     # run the simple1 tests
-    make test TEST_OPTS="-k 'simple1'"
+    python3 -m nox --session tests -- -k 'simple1'
 
     # run the simple1 tests only for XML
-    make test TEST_OPTS="-k 'xml and simple1'"
+    python3 -m nox --session tests -- -k 'xml and simple1'
 
 To see which tests would be run, add the ``--collect-only`` option:
 
 .. code:: bash
 
     #see which tests would be run
-    make test TEST_OPTS="--collect-only"
+    python3 -m nox --session tests -- --collect-only
 
 Sometimes during development you need to create reference files for new test
 or update the current reference files. To do this you have to
 add ``--generate_reference`` or ``--update-reference`` option
-to the ``TEST_OPTS`` variable.
+to the test invocation.
 By default generated output files are automatically removed after test run.
-To skip this process you can add ``--skip_clean`` option the ``TEST_OPTS``.
+To skip this process you can add ``--skip_clean`` option the test invocation.
 For example:
 
 .. code:: bash
 
     # run tests and generate references for simple1 example
-    make test TEST_OPTS="-k 'simple1' --generate_reference"
+    python3 -m nox --session tests -- -k 'simple1' --generate_reference
 
     # run tests and update xml references for simple1 example
-    make test TEST_OPTS="-k 'xml and simple1' --update_reference"
+    python3 -m nox --session tests -- -k 'xml and simple1' --update_reference
 
     # run only XML tests and do not remove generated files
-    make test TEST_OPTS="-k 'xml' --skip_clean"
+    python3 -m nox --session tests -- -k 'xml' --skip_clean
 
 When the currently generated output reports differ to the reference files
 you can create a ZIP archive named ``diff.zip`` in the tests directory
@@ -365,7 +359,10 @@ with the differences as an artifact.
 .. code:: bash
 
     # run tests and generate a ZIP archive when there were differences
-    make test TEST_OPTS="--archive_differences"
+    python3 -m nox --session tests -- --archive_differences
+
+.. versionchanged:: NEXT
+    Change how to start test from ``make test`` to ``python3 -m nox --session test``
 
 .. versionadded:: 5.0
    Added test options `--generate_reference`, `--update_reference`,
@@ -382,15 +379,21 @@ First, build the container image:
 
 .. code:: bash
 
-    make docker-qa-build
+    python3 -m nox --session docker_qa_build
+
+Then, run the container, which executes ``python3 -m nox`` within the container:
+
+.. code:: bash
+
+    python3 -m nox --session docker_qa_run
 
-Then, run the container, which executes ``make qa`` within the container:
+Or to build and run the container in one step:
 
 .. code:: bash
 
-    make docker-qa
+    python3 -m nox --session docker_qa
 
-You can select the gcc version to use inside the docker by setting the make
+You can select the gcc version to use inside the docker by setting the environment
 variable CC to gcc-5 (default), gcc-6, gcc-8 or clang-10
 
 .. _join: