From 21d056af350d7421dce361e256a0c46317c17673 Mon Sep 17 00:00:00 2001 From: "Schweizer, Robert" Date: Thu, 27 Sep 2018 10:14:14 +0200 Subject: [PATCH] Add pytest_before_assert hook. This hook will run before every assert, in contrast to the pytest_assertrepr_compare hook which only runs for failing asserts. As of now, no parameters are passed to the hook. We use this hook to print the assertion code of passed assertions to collect test evidence. This is done using inspect:: inspect.stack()[7].code_context[0].strip() Signed-off-by: Schweizer, Robert --- AUTHORS | 1 + doc/en/assert.rst | 27 ++++++++++++++++++++ doc/en/reference.rst | 5 ++-- src/_pytest/assertion/__init__.py | 17 +++++++++---- src/_pytest/assertion/rewrite.py | 7 ++++++ src/_pytest/assertion/util.py | 1 + src/_pytest/hookspec.py | 7 ++++++ testing/test_assertion.py | 42 ++++++++++++++++++++++++++++--- 8 files changed, 96 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9b4b8b8ccdf..21c7be2dd35 100644 --- a/AUTHORS +++ b/AUTHORS @@ -179,6 +179,7 @@ Raphael Pierzina Raquel Alegre Ravi Chandra Roberto Polli +Robert Schweizer Romain Dorgueil Roman Bolshakov Ronny Pfannschmidt diff --git a/doc/en/assert.rst b/doc/en/assert.rst index e0e9b930587..0af5423631b 100644 --- a/doc/en/assert.rst +++ b/doc/en/assert.rst @@ -255,6 +255,33 @@ the conftest file:: .. _assert-details: .. _`assert introspection`: +Collecting information about passing assertions +----------------------------------------------- + +The ``pytest_assertrepr_compare`` hook only runs for failing assertions. Information +about passing assertions can be collected with the ``pytest_before_assert`` hook. + +.. autofunction:: _pytest.hookspec.pytest_before_assert + :noindex: + +For example, to report every encountered assertion, the following hook +needs to be added to :ref:`conftest.py `:: + + # content of conftest.py + def pytest_before_assert(): + print("Before-assert hook is executed.") + +now, given this test module:: + + # content of test_sample.py + def test_answer(): + assert 1 == 1 + +the following stdout is captured, e.g. in an HTML report:: + + ----------------------------- Captured stdout call ----------------------------- + Before-assert hook is executed. + Advanced assertion introspection ---------------------------------- diff --git a/doc/en/reference.rst b/doc/en/reference.rst index ea0d5d7c31a..18c3d438070 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -619,9 +619,10 @@ test execution: .. autofunction:: pytest_runtest_logreport -You can also use this hook to customize assertion representation for some -types: +You can also use these hooks to output assertion information or customize +assertion representation for some types: +.. autofunction:: pytest_before_assert .. autofunction:: pytest_assertrepr_compare diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index 2c9a8890c9a..3d7363513ae 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -98,12 +98,14 @@ def pytest_collection(session): def pytest_runtest_setup(item): - """Setup the pytest_assertrepr_compare hook + """Setup the pytest_assertrepr_compare and pytest_before_assert hooks. - The newinterpret and rewrite modules will use util._reprcompare if - it exists to use custom reporting via the - pytest_assertrepr_compare hook. This sets up this custom - comparison for the test. + The rewrite module will use util._reprcompare and util._before_assert + if they exist to enable custom reporting. Comparison representation is + customized via the pytest_assertrepr_compare hook. Before every assert, + the pytest_before_assert hook is run. + + This sets up the custom assert hooks for the test. """ def callbinrepr(op, left, right): @@ -133,11 +135,16 @@ def callbinrepr(op, left, right): res = res.replace("%", "%%") return res + def call_before_assert(): + item.ihook.pytest_before_assert(config=item.config) + util._reprcompare = callbinrepr + util._before_assert = call_before_assert def pytest_runtest_teardown(item): util._reprcompare = None + util._before_assert = None def pytest_sessionfinish(session): diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py index be8c6dc4df6..7da0d402eab 100644 --- a/src/_pytest/assertion/rewrite.py +++ b/src/_pytest/assertion/rewrite.py @@ -533,6 +533,11 @@ def _call_reprcompare(ops, results, expls, each_obj): return expl +def _call_before_assert(): + if util._before_assert is not None: + util._before_assert() + + unary_map = {ast.Not: "not %s", ast.Invert: "~%s", ast.USub: "-%s", ast.UAdd: "+%s"} binop_map = { @@ -822,6 +827,8 @@ def visit_Assert(self, assert_): self.stack = [] self.on_failure = [] self.push_format_context() + # Run before assert hook. + self.statements.append(ast.Expr(self.helper("call_before_assert"))) # Rewrite assert into a bunch of statements. top_condition, explanation = self.visit(assert_.test) # Create failure message. diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index a3013cb9838..0a1356de1a7 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -14,6 +14,7 @@ # loaded and in turn call the hooks defined here as part of the # DebugInterpreter. _reprcompare = None +_before_assert = None # the re-encoding is needed for python2 repr diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 1a932614938..efafbb4f10d 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -473,6 +473,13 @@ def pytest_assertrepr_compare(config, op, left, right): """ +def pytest_before_assert(config): + """ called before every assertion is evaluated. + + :param _pytest.config.Config config: pytest config object + """ + + # ------------------------------------------------------------------------- # hooks for influencing reporting (invoked from _pytest_terminal) # ------------------------------------------------------------------------- diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 6a2a1ed3856..57c38c540fd 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -285,7 +285,7 @@ def test_register_assert_rewrite_checks_types(self): ) -class TestBinReprIntegration(object): +class TestAssertHooksIntegration(object): def test_pytest_assertrepr_compare_called(self, testdir): testdir.makeconftest( """ @@ -310,6 +310,30 @@ def test_check(list): result = testdir.runpytest("-v") result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + def test_pytest_before_assert_called(self, testdir): + testdir.makeconftest( + """ + import pytest + values = [] + def pytest_before_assert(): + values.append(True) + + @pytest.fixture + def list(request): + return values + """ + ) + testdir.makepyfile( + """ + def test_hello(): + assert 0 == 1 + def test_check(list): + assert list == [True, True] # The hook is run before the assert + """ + ) + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + def callequal(left, right, verbose=False): config = mock_config() @@ -824,18 +848,24 @@ def test_hello(): ) -def test_assertrepr_loaded_per_dir(testdir): +def test_assert_hooks_loaded_per_dir(testdir): testdir.makepyfile(test_base=["def test_base(): assert 1 == 2"]) a = testdir.mkdir("a") a_test = a.join("test_a.py") a_test.write("def test_a(): assert 1 == 2") a_conftest = a.join("conftest.py") - a_conftest.write('def pytest_assertrepr_compare(): return ["summary a"]') + a_conftest.write( + 'def pytest_before_assert(): print("assert hook a")\n' + + 'def pytest_assertrepr_compare(): return ["summary a"]' + ) b = testdir.mkdir("b") b_test = b.join("test_b.py") b_test.write("def test_b(): assert 1 == 2") b_conftest = b.join("conftest.py") - b_conftest.write('def pytest_assertrepr_compare(): return ["summary b"]') + b_conftest.write( + 'def pytest_before_assert(): print("assert hook b")\n' + + 'def pytest_assertrepr_compare(): return ["summary b"]' + ) result = testdir.runpytest() result.stdout.fnmatch_lines( [ @@ -843,8 +873,12 @@ def test_assertrepr_loaded_per_dir(testdir): "*E*assert 1 == 2*", "*def test_a():*", "*E*assert summary a*", + "*Captured stdout call*", + "*assert hook a*", "*def test_b():*", "*E*assert summary b*", + "*Captured stdout call*", + "*assert hook b*", ] )