From cfcf9821a57efadee1494e7a36b766d707688b9c Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Thu, 25 Nov 2021 19:58:21 +1100 Subject: [PATCH 1/3] Defer plugin imports --- hypothesis-python/.coveragerc | 2 +- hypothesis-python/RELEASE.rst | 8 ++ hypothesis-python/docs/strategies.rst | 9 +- hypothesis-python/setup.py | 3 +- ...tplugin.py => _hypothesis_pytestplugin.py} | 93 ++++++++++++++----- .../tests/pytest/test_profiles.py | 2 +- .../tests/pytest/test_pytest_detection.py | 13 +++ .../tests/pytest/test_statistics.py | 3 +- 8 files changed, 103 insertions(+), 30 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst rename hypothesis-python/src/{hypothesis/extra/pytestplugin.py => _hypothesis_pytestplugin.py} (83%) diff --git a/hypothesis-python/.coveragerc b/hypothesis-python/.coveragerc index ef37f8d774..baa88b2536 100644 --- a/hypothesis-python/.coveragerc +++ b/hypothesis-python/.coveragerc @@ -1,7 +1,7 @@ [run] branch = True omit = - **/extra/pytestplugin.py + **/_hypothesis_pytestplugin.py **/extra/array_api.py **/extra/cli.py **/extra/django/*.py diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..5bc62a7a9c --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,8 @@ +RELEASE_TYPE: minor + +This release modifies our :pypi:`pytest` plugin, to avoid importing Hypothesis +and therefore triggering :ref:`Hypothesis' entry points ` for +test suites where Hypothesis is installed but not actually used (:issue:`3140`). + +If you :ref:`manually load the plugin `, you'll need +to update the module name accordingly. diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index 781e5f713e..25b76a3455 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -128,13 +128,14 @@ test is using Hypothesis: .. _entry-points: -------------------------------------------------- -Registering strategies via setuptools entry points +Hypothesis integration via setuptools entry points -------------------------------------------------- If you would like to ship Hypothesis strategies for a custom type - either as part of the upstream library, or as a third-party extension, there's a catch: :func:`~hypothesis.strategies.from_type` only works after the corresponding -call to :func:`~hypothesis.strategies.register_type_strategy`. This means that +call to :func:`~hypothesis.strategies.register_type_strategy`, and you'll have +the same problem with :func:`~hypothesis.register_random`. This means that either - you have to try importing Hypothesis to register the strategy when *your* @@ -188,6 +189,8 @@ And that's all it takes! package to be installed. +.. _disabling-pytest-plugin: + Interaction with :pypi:`pytest-cov` ----------------------------------- @@ -201,5 +204,5 @@ opting out of the pytest plugin entirely. Alternatively, you can ensure that Hy is loaded after coverage measurement is started by disabling the entrypoint, and loading our pytest plugin from your ``conftest.py`` instead:: - echo "pytest_plugins = ['hypothesis.extra.pytestplugin']\n" > tests/conftest.py + echo "pytest_plugins = ['_hypothesis_pytestplugin']\n" > tests/conftest.py pytest -p "no:hypothesispytest" ... diff --git a/hypothesis-python/setup.py b/hypothesis-python/setup.py index 32a5817bfd..6a73a57285 100644 --- a/hypothesis-python/setup.py +++ b/hypothesis-python/setup.py @@ -129,8 +129,9 @@ def local_file(name): "Topic :: Software Development :: Testing", "Typing :: Typed", ], + py_modules=["_hypothesis_pytestplugin"], entry_points={ - "pytest11": ["hypothesispytest = hypothesis.extra.pytestplugin"], + "pytest11": ["hypothesispytest = _hypothesis_pytestplugin"], "console_scripts": ["hypothesis = hypothesis.extra.cli:main"], }, long_description=open(README).read(), diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py similarity index 83% rename from hypothesis-python/src/hypothesis/extra/pytestplugin.py rename to hypothesis-python/src/_hypothesis_pytestplugin.py index 767e2a3651..1bf16596e1 100644 --- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -13,34 +13,51 @@ # # END HEADER +""" +The pytest plugin for Hypothesis. + +We move this from the old location at `hypothesis.extra.pytestplugin` so that it +can be loaded by Pytest without importing Hypothesis. In turn, this means that +Hypothesis will not load our own third-party plugins (with associated side-effects) +unless and until the user explicitly runs `import hypothesis`. + +See https://github.com/HypothesisWorks/hypothesis/issues/3140 for details. +""" + import base64 +import sys from inspect import signature import pytest -from hypothesis import HealthCheck, Phase, Verbosity, core, settings -from hypothesis.errors import InvalidArgument -from hypothesis.internal.detection import is_hypothesis_test -from hypothesis.internal.escalation import current_pytest_item -from hypothesis.internal.healthcheck import fail_health_check -from hypothesis.reporting import default as default_reporter, with_reporter -from hypothesis.statistics import collector, describe_statistics - LOAD_PROFILE_OPTION = "--hypothesis-profile" VERBOSITY_OPTION = "--hypothesis-verbosity" PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics" SEED_OPTION = "--hypothesis-seed" EXPLAIN_OPTION = "--hypothesis-explain" +_VERBOSITY_NAMES = ["quiet", "normal", "verbose", "debug"] +_ALL_OPTIONS = [ + LOAD_PROFILE_OPTION, + VERBOSITY_OPTION, + PRINT_STATISTICS_OPTION, + SEED_OPTION, + EXPLAIN_OPTION, +] + class StoringReporter: def __init__(self, config): + assert "hypothesis" in sys.modules + from hypothesis.reporting import default + + self.report = default self.config = config self.results = [] def __call__(self, msg): if self.config.getoption("capture", "fd") == "no": - default_reporter(msg) + self.report(msg) if not isinstance(msg, str): msg = repr(msg) self.results.append(msg) @@ -51,15 +68,13 @@ def __call__(self, msg): if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6): # pragma: no cover import warnings - from hypothesis.errors import HypothesisWarning - PYTEST_TOO_OLD_MESSAGE = """ You are using pytest version %s. Hypothesis tests work with any test runner, but our pytest plugin requires pytest 4.6 or newer. Note that the pytest developers no longer support your version either! Disabling the Hypothesis pytest plugin... """ - warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,), HypothesisWarning) + warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,)) else: @@ -73,7 +88,7 @@ def pytest_addoption(parser): group.addoption( VERBOSITY_OPTION, action="store", - choices=[opt.name for opt in Verbosity], + choices=_VERBOSITY_NAMES, help="Override profile with verbosity setting specified", ) group.addoption( @@ -94,19 +109,32 @@ def pytest_addoption(parser): default=False, ) + def _any_hypothesis_option(config): + return bool(any(config.getoption(opt) for opt in _ALL_OPTIONS)) + def pytest_report_header(config): + if not ( + config.option.verbose >= 1 + or "hypothesis" in sys.modules + or _any_hypothesis_option(config) + ): + return None + + from hypothesis import Verbosity, settings + if config.option.verbose < 1 and settings.default.verbosity < Verbosity.verbose: return None - profile = config.getoption(LOAD_PROFILE_OPTION) - if not profile: - profile = settings._current_profile - settings_str = settings.get_profile(profile).show_changed() + settings_str = settings.default.show_changed() if settings_str != "": settings_str = f" -> {settings_str}" - return f"hypothesis profile {profile!r}{settings_str}" + return f"hypothesis profile {settings._current_profile!r}{settings_str}" def pytest_configure(config): - core.running_under_pytest = True + config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.") + if not _any_hypothesis_option(config): + return + from hypothesis import Phase, Verbosity, core, settings + profile = config.getoption(LOAD_PROFILE_OPTION) if profile: settings.load_profile(profile) @@ -134,16 +162,24 @@ def pytest_configure(config): except ValueError: pass core.global_force_seed = seed - config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.") @pytest.hookimpl(hookwrapper=True) def pytest_runtest_call(item): - if not hasattr(item, "obj"): + if not (hasattr(item, "obj") and "hypothesis" in sys.modules): yield - elif not is_hypothesis_test(item.obj): + return + + from hypothesis import core + from hypothesis.internal.detection import is_hypothesis_test + + core.running_under_pytest = True + + if not is_hypothesis_test(item.obj): # If @given was not applied, check whether other hypothesis # decorators were applied, and raise an error if they were. if getattr(item.obj, "is_hypothesis_strategy_function", False): + from hypothesis.errors import InvalidArgument + raise InvalidArgument( f"{item.nodeid} is a function that returns a Hypothesis strategy, " "but pytest has collected it as a test function. This is useless " @@ -158,9 +194,17 @@ def pytest_runtest_call(item): ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), ]: if hasattr(item.obj, attribute): + from hypothesis.errors import InvalidArgument + raise InvalidArgument(message % (name,)) yield else: + from hypothesis import HealthCheck, settings + from hypothesis.internal.escalation import current_pytest_item + from hypothesis.internal.healthcheck import fail_health_check + from hypothesis.reporting import with_reporter + from hypothesis.statistics import collector, describe_statistics + # Retrieve the settings for this test from the test object, which # is normally a Hypothesis wrapped_test wrapper. If this doesn't # work, the test object is probably something weird @@ -273,6 +317,11 @@ def report(properties): report(global_properties) def pytest_collection_modifyitems(items): + if "hypothesis" not in sys.modules: + return + + from hypothesis.internal.detection import is_hypothesis_test + for item in items: if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj): item.add_marker("hypothesis") diff --git a/hypothesis-python/tests/pytest/test_profiles.py b/hypothesis-python/tests/pytest/test_profiles.py index 4150b6a605..b2cb7b21d2 100644 --- a/hypothesis-python/tests/pytest/test_profiles.py +++ b/hypothesis-python/tests/pytest/test_profiles.py @@ -14,8 +14,8 @@ # END HEADER import pytest +from _hypothesis_pytestplugin import LOAD_PROFILE_OPTION -from hypothesis.extra.pytestplugin import LOAD_PROFILE_OPTION from hypothesis.version import __version__ pytest_plugins = "pytester" diff --git a/hypothesis-python/tests/pytest/test_pytest_detection.py b/hypothesis-python/tests/pytest/test_pytest_detection.py index 9319d30c82..ccf9fee884 100644 --- a/hypothesis-python/tests/pytest/test_pytest_detection.py +++ b/hypothesis-python/tests/pytest/test_pytest_detection.py @@ -33,3 +33,16 @@ def test_is_not_running_under_pytest(tmpdir): pyfile = tmpdir.join("test.py") pyfile.write(FILE_TO_RUN) subprocess.check_call([sys.executable, str(pyfile)]) + + +DOES_NOT_IMPORT_HYPOTHESIS = """ +import sys + +def test_pytest_plugin_does_not_import_hypothesis(): + assert "hypothesis" not in sys.modules +""" + + +def test_plugin_does_not_import_pytest(testdir): + testdir.makepyfile(DOES_NOT_IMPORT_HYPOTHESIS) + testdir.runpytest_subprocess().assert_outcomes(passed=1) diff --git a/hypothesis-python/tests/pytest/test_statistics.py b/hypothesis-python/tests/pytest/test_statistics.py index 73dc3ab1a1..d958be20c6 100644 --- a/hypothesis-python/tests/pytest/test_statistics.py +++ b/hypothesis-python/tests/pytest/test_statistics.py @@ -16,8 +16,7 @@ from distutils.version import LooseVersion import pytest - -from hypothesis.extra.pytestplugin import PRINT_STATISTICS_OPTION +from _hypothesis_pytestplugin import PRINT_STATISTICS_OPTION pytest_plugins = "pytester" From 221aa1ad070a58d5ef2e96dec18cbdfb56b17a5e Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Thu, 25 Nov 2021 19:58:21 +1100 Subject: [PATCH 2/3] Add stub pytestplugin --- hypothesis-python/.coveragerc | 1 + hypothesis-python/RELEASE.rst | 3 --- hypothesis-python/docs/strategies.rst | 4 +--- .../src/hypothesis/extra/pytestplugin.py | 24 +++++++++++++++++++ 4 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 hypothesis-python/src/hypothesis/extra/pytestplugin.py diff --git a/hypothesis-python/.coveragerc b/hypothesis-python/.coveragerc index baa88b2536..b5a21591f8 100644 --- a/hypothesis-python/.coveragerc +++ b/hypothesis-python/.coveragerc @@ -6,6 +6,7 @@ omit = **/extra/cli.py **/extra/django/*.py **/extra/ghostwriter.py + **/extra/pytestplugin.py **/internal/scrutineer.py **/utils/terminal.py diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index 5bc62a7a9c..7727d5ad9a 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -3,6 +3,3 @@ RELEASE_TYPE: minor This release modifies our :pypi:`pytest` plugin, to avoid importing Hypothesis and therefore triggering :ref:`Hypothesis' entry points ` for test suites where Hypothesis is installed but not actually used (:issue:`3140`). - -If you :ref:`manually load the plugin `, you'll need -to update the module name accordingly. diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index 25b76a3455..4918da61fc 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -189,8 +189,6 @@ And that's all it takes! package to be installed. -.. _disabling-pytest-plugin: - Interaction with :pypi:`pytest-cov` ----------------------------------- @@ -204,5 +202,5 @@ opting out of the pytest plugin entirely. Alternatively, you can ensure that Hy is loaded after coverage measurement is started by disabling the entrypoint, and loading our pytest plugin from your ``conftest.py`` instead:: - echo "pytest_plugins = ['_hypothesis_pytestplugin']\n" > tests/conftest.py + echo "pytest_plugins = ['hypothesis.extra.pytestplugin']\n" > tests/conftest.py pytest -p "no:hypothesispytest" ... diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py new file mode 100644 index 0000000000..2f160a94cf --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py @@ -0,0 +1,24 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Most of this work is copyright (C) 2013-2021 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. +# +# END HEADER + +""" +Stub for users who manually load our pytest plugin. + +The plugin implementation is now located in a top-level module outside the main +hypothesis tree, so that Pytest can load the plugin without thereby triggering +the import of Hypothesis itself (and thus loading our own plugins). +""" + +from _hypothesis_pytestplugin import * # noqa From 616b2f36d528ce7d6a34b7f1c72b42386584c36c Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Fri, 26 Nov 2021 12:42:15 +1100 Subject: [PATCH 3/3] Improve fixture message This isn't perfect, but it seems like a big improvement over the status quo. Closes #3018. Co-Authored-By: Stuart Cook --- .../src/_hypothesis_pytestplugin.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py index 1bf16596e1..27085bdf3d 100644 --- a/hypothesis-python/src/_hypothesis_pytestplugin.py +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -44,6 +44,19 @@ SEED_OPTION, EXPLAIN_OPTION, ] +_FIXTURE_MSG = """Function-scoped fixture {0!r} used by {1!r} + +Function-scoped fixtures are not reset between examples generated by +`@given(...)`, which is often surprising and can cause subtle test bugs. + +If you were expecting the fixture to run separately for each generated example, +then unfortunately you will need to find a different way to achieve your goal +(e.g. using a similar context manager instead of a fixture). + +If you are confident that your test will work correctly even though the +fixture is not reset between generated examples, you can suppress this health +check to assure Hypothesis that you understand what you are doing. +""" class StoringReporter: @@ -222,15 +235,6 @@ def pytest_runtest_call(item): # Warn about function-scoped fixtures, excluding autouse fixtures because # the advice is probably not actionable and the status quo seems OK... # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. - msg = ( - "%s uses the %r fixture, which is reset between function calls but not " - "between test cases generated by `@given(...)`. You can change it to " - "a module- or session-scoped fixture if it is safe to reuse; if not " - "we recommend using a context manager inside your test function. See " - "https://docs.pytest.org/en/latest/how-to/fixtures.html" - "#scope-sharing-fixtures-across-classes-modules-packages-or-session " - "for details on fixture scope." - ) argnames = None for fx_defs in item._request._fixturemanager.getfixtureinfo( node=item, func=item.function, cls=None @@ -243,7 +247,7 @@ def pytest_runtest_call(item): if active_fx.scope == "function": fail_health_check( settings, - msg % (item.nodeid, fx.argname), + _FIXTURE_MSG.format(fx.argname, item.nodeid), HealthCheck.function_scoped_fixture, )