Add progress trackers during test execution (#205)

* add a progress bar during test execution

* allow for multiple styles; add inline style

* cleanup progress trackers

* respond to review comments; upgrade to Rich v10; add CLI help text and option compatibility checking

* Minor styling changes

* Fix merge issue

* Rename private method since its used publicly now

* move unparameterised test length determination inside find_number_of_instances

* move test_cli to the right directory

* max_dots_per_line needs to reset when we jump to a new line

* switch order of printing new dot and end-of-line detection to fix double-printing end-of-line when the number of tests is equal to the max number of dots for the line

* Adding docs for progress bar and inline test progress styles

Co-authored-by: Darren Burns <darrenb900@gmail.com>

docs/source/guide/running_tests.rst

@@ -122,6 +122,26 @@ If that is still too verbose, you may wish to represent every test outcome with
     :align: center
     :alt: Output using dots-global mode
 
+Displaying test session progress with ``--progress-style``
+----------------------------------------------------------
+
+Ward offers two ways of informing you of progress through a test run: inline progress percentage (on by default), and/or a dynamic progress bar.
+
+By default, the percentage progress through a test run will appear at the right hand side of the output, which corresponds to ``--progress-style inline``.
+
+You can also have Ward display a dynamic progress bar during the test run, using the ``--progress-style bar`` option.
+
+.. image:: ../_static/ward_progress_bar.gif
+    :align: center
+    :alt: Example of progress-style of bar
+
+If you wish, can pass supply ``--progress-style`` with multiple times (to display a progress bar and inline progress, for example).
+
+.. warning::
+
+    The progress bar is currently only available with the default output mode (``--test-output-style test-per-line``).
+
+
 Output capturing
 ----------------
 

pyproject.toml

@@ -40,7 +40,7 @@ dataclasses = { version = "^0.7", python = "3.6" }
 colorama = "^0.4.0"
 click = "^7.0"
 termcolor = "^1.1.0"
-rich = "^9.1.0"
+rich = "^10.0.0"
 toml = "^0.10.1"
 pygments = "^2.4.2"
 pprintpp = "^0.4.0"

tests/test_cli.py

@@ -0,0 +1,14 @@
+from click.testing import CliRunner
+
+from ward import each, test
+from ward.run import run
+
+
+@test("Cannot use bar progress style with {output_style} output style")
+def _(output_style=each("dots-global", "dots-module")):
+    runner = CliRunner()
+    result = runner.invoke(
+        run, ["test", "--progress-style", "bar", "--test-output-style", output_style]
+    )
+
+    assert result.exit_code == 2

ward/run.py

@@ -3,7 +3,7 @@
 import sys
 from pathlib import Path
 from timeit import default_timer
-from typing import Optional, Tuple
+from typing import Optional, Tuple, List
 
 import click
 import click_completion
@@ -25,7 +25,13 @@
 from ward.rewrite import rewrite_assertions_in_tests
 from ward.suite import Suite
 from ward.fixtures import _DEFINED_FIXTURES
-from ward.terminal import SimpleTestResultWrite, output_fixtures, get_exit_code
+from ward.terminal import (
+    SimpleTestResultWrite,
+    output_fixtures,
+    get_exit_code,
+    TestProgressStyle,
+    TestOutputStyle,
+)
 
 colorama.init()
 click_completion.init()
@@ -91,10 +97,20 @@ def run(ctx: click.Context):
 )
 @click.option(
     "--test-output-style",
-    type=click.Choice(
-        ["test-per-line", "dots-global", "dots-module"], case_sensitive=False,
-    ),
+    type=click.Choice(list(TestOutputStyle), case_sensitive=False),
     default="test-per-line",
+    help="The style of output for displaying individual test results during the run.",
+)
+@click.option(
+    "--progress-style",
+    type=click.Choice(list(TestProgressStyle), case_sensitive=False),
+    multiple=True,
+    default=["inline"],
+    help=f"""\
+    The style of progress indicator to use during the run.
+    Pass multiple times to enable multiple styles.
+    The '{TestProgressStyle.BAR}' style is not compatible with the '{TestOutputStyle.DOTS_GLOBAL}' and '{TestOutputStyle.DOTS_MODULE}' test output styles.
+    """,
 )
 @click.option(
     "--order",
@@ -135,13 +151,25 @@ def test(
     tags: Optional[Expression],
     fail_limit: Optional[int],
     test_output_style: str,
+    progress_style: List[str],
     order: str,
     capture_output: bool,
     show_slowest: int,
     show_diff_symbols: bool,
     dry_run: bool,
 ):
     """Run tests."""
+    progress_styles = [TestProgressStyle(ps) for ps in progress_style]
+
+    if TestProgressStyle.BAR in progress_styles and test_output_style in {
+        "dots-global",
+        "dots-module",
+    }:
+        raise click.BadOptionUsage(
+            "progress_style",
+            f"The '{TestProgressStyle.BAR}' progress style cannot be used with dots-based test output styles (you asked for '{test_output_style}').",
+        )
+
     init_breakpointhooks(pdb, sys)
     start_run = default_timer()
     paths = [Path(p) for p in path]
@@ -160,6 +188,7 @@ def test(
     writer = SimpleTestResultWrite(
         suite=suite,
         test_output_style=test_output_style,
+        progress_styles=progress_styles,
         config_path=config_path,
         show_diff_symbols=show_diff_symbols,
     )

ward/suite.py

@@ -15,9 +15,13 @@ class Suite:
     cache: FixtureCache = field(default_factory=FixtureCache)
 
     @property
-    def num_tests(self):
+    def num_tests(self) -> int:
         return len(self.tests)
 
+    @property
+    def num_tests_with_parameterisation(self) -> int:
+        return sum(test.find_number_of_instances() for test in self.tests)
+
     def _test_counts_per_module(self):
         module_paths = [test.path for test in self.tests]
         counts = defaultdict(int)