From df0a559e670a2661befb10e8e0de326225a5c96f Mon Sep 17 00:00:00 2001 From: "Schweizer, Robert" Date: Thu, 27 Sep 2018 10:14:14 +0200 Subject: [PATCH] Add pytest_before_assert hook. This hook will run before every assert, in contrast to the pytest_assertrepr_compare hook which only runs for failing asserts. As of now, no parameters are passed to the hook. We use this hook to print the assertion code of passed assertions to collect test evidence. This is done using inspect:: inspect.stack()[7].code_context[0].strip() Signed-off-by: Schweizer, Robert --- AUTHORS | 1 + changelog/4047.feature.rst | 1 + doc/en/assert.rst | 27 ++++++++++++++++++++ doc/en/reference.rst | 5 ++-- src/_pytest/assertion/__init__.py | 17 +++++++++---- src/_pytest/assertion/rewrite.py | 7 ++++++ src/_pytest/assertion/util.py | 1 + src/_pytest/hookspec.py | 7 ++++++ testing/test_assertion.py | 42 ++++++++++++++++++++++++++++--- 9 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 changelog/4047.feature.rst diff --git a/AUTHORS b/AUTHORS index 85fe6aff01a..341a3dedb9e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -200,6 +200,7 @@ Raphael Pierzina Raquel Alegre Ravi Chandra Roberto Polli +Robert Schweizer Roland Puntaier Romain Dorgueil Roman Bolshakov diff --git a/changelog/4047.feature.rst b/changelog/4047.feature.rst new file mode 100644 index 00000000000..c240af8164c --- /dev/null +++ b/changelog/4047.feature.rst @@ -0,0 +1 @@ +Add pytest_before_assert hook to run before any assertion. diff --git a/doc/en/assert.rst b/doc/en/assert.rst index e7e78601b06..27e40c33330 100644 --- a/doc/en/assert.rst +++ b/doc/en/assert.rst @@ -252,6 +252,33 @@ the conftest file: .. _assert-details: .. _`assert introspection`: +Collecting information about passing assertions +----------------------------------------------- + +The ``pytest_assertrepr_compare`` hook only runs for failing assertions. Information +about passing assertions can be collected with the ``pytest_before_assert`` hook. + +.. autofunction:: _pytest.hookspec.pytest_before_assert + :noindex: + +For example, to report every encountered assertion, the following hook +needs to be added to :ref:`conftest.py `:: + + # content of conftest.py + def pytest_before_assert(): + print("Before-assert hook is executed.") + +now, given this test module:: + + # content of test_sample.py + def test_answer(): + assert 1 == 1 + +the following stdout is captured, e.g. in an HTML report:: + + ----------------------------- Captured stdout call ----------------------------- + Before-assert hook is executed. + Assertion introspection details ------------------------------- diff --git a/doc/en/reference.rst b/doc/en/reference.rst index 7ab7340752f..6c23688afba 100644 --- a/doc/en/reference.rst +++ b/doc/en/reference.rst @@ -651,9 +651,10 @@ test execution: .. autofunction:: pytest_runtest_logreport -You can also use this hook to customize assertion representation for some -types: +You can also use these hooks to output assertion information or customize +assertion representation for some types: +.. autofunction:: pytest_before_assert .. autofunction:: pytest_assertrepr_compare diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index b5c846c2c00..7d4486e51d6 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -102,12 +102,14 @@ def pytest_collection(session): def pytest_runtest_setup(item): - """Setup the pytest_assertrepr_compare hook + """Setup the pytest_assertrepr_compare and pytest_before_assert hooks. - The newinterpret and rewrite modules will use util._reprcompare if - it exists to use custom reporting via the - pytest_assertrepr_compare hook. This sets up this custom - comparison for the test. + The rewrite module will use util._reprcompare and util._before_assert + if they exist to enable custom reporting. Comparison representation is + customized via the pytest_assertrepr_compare hook. Before every assert, + the pytest_before_assert hook is run. + + This sets up the custom assert hooks for the test. """ def callbinrepr(op, left, right): @@ -137,11 +139,16 @@ def callbinrepr(op, left, right): res = res.replace("%", "%%") return res + def call_before_assert(): + item.ihook.pytest_before_assert(config=item.config) + util._reprcompare = callbinrepr + util._before_assert = call_before_assert def pytest_runtest_teardown(item): util._reprcompare = None + util._before_assert = None def pytest_sessionfinish(session): diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py index 19192c9d907..d0501db31a0 100644 --- a/src/_pytest/assertion/rewrite.py +++ b/src/_pytest/assertion/rewrite.py @@ -545,6 +545,11 @@ def _call_reprcompare(ops, results, expls, each_obj): return expl +def _call_before_assert(): + if util._before_assert is not None: + util._before_assert() + + unary_map = {ast.Not: "not %s", ast.Invert: "~%s", ast.USub: "-%s", ast.UAdd: "+%s"} binop_map = { @@ -833,6 +838,8 @@ def visit_Assert(self, assert_): self.stack = [] self.on_failure = [] self.push_format_context() + # Run before assert hook. + self.statements.append(ast.Expr(self.helper("call_before_assert"))) # Rewrite assert into a bunch of statements. top_condition, explanation = self.visit(assert_.test) # If in a test module, check if directly asserting None, in order to warn [Issue #3191] diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 6326dddbdc4..a336c7d39cc 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -16,6 +16,7 @@ # loaded and in turn call the hooks defined here as part of the # DebugInterpreter. _reprcompare = None +_before_assert = None # the re-encoding is needed for python2 repr diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 0641e3bc5ac..20ec553bda7 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -446,6 +446,13 @@ def pytest_assertrepr_compare(config, op, left, right): """ +def pytest_before_assert(config): + """ called before every assertion is evaluated. + + :param _pytest.config.Config config: pytest config object + """ + + # ------------------------------------------------------------------------- # hooks for influencing reporting (invoked from _pytest_terminal) # ------------------------------------------------------------------------- diff --git a/testing/test_assertion.py b/testing/test_assertion.py index e4fe56c6fdf..37375a3f828 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -290,7 +290,7 @@ def test_register_assert_rewrite_checks_types(self): ) -class TestBinReprIntegration(object): +class TestAssertHooksIntegration(object): def test_pytest_assertrepr_compare_called(self, testdir): testdir.makeconftest( """ @@ -315,6 +315,30 @@ def test_check(list): result = testdir.runpytest("-v") result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + def test_pytest_before_assert_called(self, testdir): + testdir.makeconftest( + """ + import pytest + values = [] + def pytest_before_assert(): + values.append(True) + + @pytest.fixture + def list(request): + return values + """ + ) + testdir.makepyfile( + """ + def test_hello(): + assert 0 == 1 + def test_check(list): + assert list == [True, True] # The hook is run before the assert + """ + ) + result = testdir.runpytest("-v") + result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + def callequal(left, right, verbose=False): config = mock_config() @@ -962,18 +986,24 @@ def test_hello(): ) -def test_assertrepr_loaded_per_dir(testdir): +def test_assert_hooks_loaded_per_dir(testdir): testdir.makepyfile(test_base=["def test_base(): assert 1 == 2"]) a = testdir.mkdir("a") a_test = a.join("test_a.py") a_test.write("def test_a(): assert 1 == 2") a_conftest = a.join("conftest.py") - a_conftest.write('def pytest_assertrepr_compare(): return ["summary a"]') + a_conftest.write( + 'def pytest_before_assert(): print("assert hook a")\n' + + 'def pytest_assertrepr_compare(): return ["summary a"]' + ) b = testdir.mkdir("b") b_test = b.join("test_b.py") b_test.write("def test_b(): assert 1 == 2") b_conftest = b.join("conftest.py") - b_conftest.write('def pytest_assertrepr_compare(): return ["summary b"]') + b_conftest.write( + 'def pytest_before_assert(): print("assert hook b")\n' + + 'def pytest_assertrepr_compare(): return ["summary b"]' + ) result = testdir.runpytest() result.stdout.fnmatch_lines( [ @@ -981,8 +1011,12 @@ def test_assertrepr_loaded_per_dir(testdir): "*E*assert 1 == 2*", "*def test_a():*", "*E*assert summary a*", + "*Captured stdout call*", + "*assert hook a*", "*def test_b():*", "*E*assert summary b*", + "*Captured stdout call*", + "*assert hook b*", ] )