Defer plugin imports

hypothesis-python/.coveragerc

@@ -1,11 +1,12 @@
 [run]
 branch = True
 omit =
-    **/extra/pytestplugin.py
+    **/_hypothesis_pytestplugin.py
     **/extra/array_api.py
     **/extra/cli.py
     **/extra/django/*.py
     **/extra/ghostwriter.py
+    **/extra/pytestplugin.py
     **/internal/scrutineer.py
     **/utils/terminal.py
 

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: minor
+
+This release modifies our :pypi:`pytest` plugin, to avoid importing Hypothesis
+and therefore triggering :ref:`Hypothesis' entry points <entry-points>` for
+test suites where Hypothesis is installed but not actually used (:issue:`3140`).

hypothesis-python/docs/strategies.rst

@@ -128,13 +128,14 @@ test is using Hypothesis:
 .. _entry-points:
 
 --------------------------------------------------
-Registering strategies via setuptools entry points
+Hypothesis integration via setuptools entry points
 --------------------------------------------------
 
 If you would like to ship Hypothesis strategies for a custom type - either as
 part of the upstream library, or as a third-party extension, there's a catch:
 :func:`~hypothesis.strategies.from_type` only works after the corresponding
-call to :func:`~hypothesis.strategies.register_type_strategy`.  This means that
+call to :func:`~hypothesis.strategies.register_type_strategy`, and you'll have
+the same problem with :func:`~hypothesis.register_random`.  This means that
 either
 
 - you have to try importing Hypothesis to register the strategy when *your*

hypothesis-python/setup.py

@@ -129,8 +129,9 @@ def local_file(name):
         "Topic :: Software Development :: Testing",
         "Typing :: Typed",
     ],
+    py_modules=["_hypothesis_pytestplugin"],
     entry_points={
-        "pytest11": ["hypothesispytest = hypothesis.extra.pytestplugin"],
+        "pytest11": ["hypothesispytest = _hypothesis_pytestplugin"],
         "console_scripts": ["hypothesis = hypothesis.extra.cli:main"],
     },
     long_description=open(README).read(),

hypothesis-python/src/_hypothesis_pytestplugin.py

@@ -0,0 +1,335 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2021 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+"""
+The pytest plugin for Hypothesis.
+
+We move this from the old location at `hypothesis.extra.pytestplugin` so that it
+can be loaded by Pytest without importing Hypothesis.  In turn, this means that
+Hypothesis will not load our own third-party plugins (with associated side-effects)
+unless and until the user explicitly runs `import hypothesis`.
+
+See https://github.com/HypothesisWorks/hypothesis/issues/3140 for details.
+"""
+
+import base64
+import sys
+from inspect import signature
+
+import pytest
+
+LOAD_PROFILE_OPTION = "--hypothesis-profile"
+VERBOSITY_OPTION = "--hypothesis-verbosity"
+PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics"
+SEED_OPTION = "--hypothesis-seed"
+EXPLAIN_OPTION = "--hypothesis-explain"
+
+_VERBOSITY_NAMES = ["quiet", "normal", "verbose", "debug"]
+_ALL_OPTIONS = [
+    LOAD_PROFILE_OPTION,
+    VERBOSITY_OPTION,
+    PRINT_STATISTICS_OPTION,
+    SEED_OPTION,
+    EXPLAIN_OPTION,
+]
+_FIXTURE_MSG = """Function-scoped fixture {0!r} used by {1!r}
+
+Function-scoped fixtures are not reset between examples generated by
+`@given(...)`, which is often surprising and can cause subtle test bugs.
+
+If you were expecting the fixture to run separately for each generated example,
+then unfortunately you will need to find a different way to achieve your goal
+(e.g. using a similar context manager instead of a fixture).
+
+If you are confident that your test will work correctly even though the
+fixture is not reset between generated examples, you can suppress this health
+check to assure Hypothesis that you understand what you are doing.
+"""
+
+
+class StoringReporter:
+    def __init__(self, config):
+        assert "hypothesis" in sys.modules
+        from hypothesis.reporting import default
+
+        self.report = default
+        self.config = config
+        self.results = []
+
+    def __call__(self, msg):
+        if self.config.getoption("capture", "fd") == "no":
+            self.report(msg)
+        if not isinstance(msg, str):
+            msg = repr(msg)
+        self.results.append(msg)
+
+
+# Avoiding distutils.version.LooseVersion due to
+# https://github.com/HypothesisWorks/hypothesis/issues/2490
+if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6):  # pragma: no cover
+    import warnings
+
+    PYTEST_TOO_OLD_MESSAGE = """
+        You are using pytest version %s. Hypothesis tests work with any test
+        runner, but our pytest plugin requires pytest 4.6 or newer.
+        Note that the pytest developers no longer support your version either!
+        Disabling the Hypothesis pytest plugin...
+    """
+    warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,))
+
+else:
+
+    def pytest_addoption(parser):
+        group = parser.getgroup("hypothesis", "Hypothesis")
+        group.addoption(
+            LOAD_PROFILE_OPTION,
+            action="store",
+            help="Load in a registered hypothesis.settings profile",
+        )
+        group.addoption(
+            VERBOSITY_OPTION,
+            action="store",
+            choices=_VERBOSITY_NAMES,
+            help="Override profile with verbosity setting specified",
+        )
+        group.addoption(
+            PRINT_STATISTICS_OPTION,
+            action="store_true",
+            help="Configure when statistics are printed",
+            default=False,
+        )
+        group.addoption(
+            SEED_OPTION,
+            action="store",
+            help="Set a seed to use for all Hypothesis tests",
+        )
+        group.addoption(
+            EXPLAIN_OPTION,
+            action="store_true",
+            help="Enable the `explain` phase for failing Hypothesis tests",
+            default=False,
+        )
+
+    def _any_hypothesis_option(config):
+        return bool(any(config.getoption(opt) for opt in _ALL_OPTIONS))
+
+    def pytest_report_header(config):
+        if not (
+            config.option.verbose >= 1
+            or "hypothesis" in sys.modules
+            or _any_hypothesis_option(config)
+        ):
+            return None
+
+        from hypothesis import Verbosity, settings
+
+        if config.option.verbose < 1 and settings.default.verbosity < Verbosity.verbose:
+            return None
+        settings_str = settings.default.show_changed()
+        if settings_str != "":
+            settings_str = f" -> {settings_str}"
+        return f"hypothesis profile {settings._current_profile!r}{settings_str}"
+
+    def pytest_configure(config):
+        config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.")
+        if not _any_hypothesis_option(config):
+            return
+        from hypothesis import Phase, Verbosity, core, settings
+
+        profile = config.getoption(LOAD_PROFILE_OPTION)
+        if profile:
+            settings.load_profile(profile)
+        verbosity_name = config.getoption(VERBOSITY_OPTION)
+        if verbosity_name and verbosity_name != settings.default.verbosity.name:
+            verbosity_value = Verbosity[verbosity_name]
+            name = f"{settings._current_profile}-with-{verbosity_name}-verbosity"
+            # register_profile creates a new profile, exactly like the current one,
+            # with the extra values given (in this case 'verbosity')
+            settings.register_profile(name, verbosity=verbosity_value)
+            settings.load_profile(name)
+        if (
+            config.getoption(EXPLAIN_OPTION)
+            and Phase.explain not in settings.default.phases
+        ):
+            name = f"{settings._current_profile}-with-explain-phase"
+            phases = settings.default.phases + (Phase.explain,)
+            settings.register_profile(name, phases=phases)
+            settings.load_profile(name)
+
+        seed = config.getoption(SEED_OPTION)
+        if seed is not None:
+            try:
+                seed = int(seed)
+            except ValueError:
+                pass
+            core.global_force_seed = seed
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_call(item):
+        if not (hasattr(item, "obj") and "hypothesis" in sys.modules):
+            yield
+            return
+
+        from hypothesis import core
+        from hypothesis.internal.detection import is_hypothesis_test
+
+        core.running_under_pytest = True
+
+        if not is_hypothesis_test(item.obj):
+            # If @given was not applied, check whether other hypothesis
+            # decorators were applied, and raise an error if they were.
+            if getattr(item.obj, "is_hypothesis_strategy_function", False):
+                from hypothesis.errors import InvalidArgument
+
+                raise InvalidArgument(
+                    f"{item.nodeid} is a function that returns a Hypothesis strategy, "
+                    "but pytest has collected it as a test function.  This is useless "
+                    "as the function body will never be executed.  To define a test "
+                    "function, use @given instead of @composite."
+                )
+            message = "Using `@%s` on a test without `@given` is completely pointless."
+            for name, attribute in [
+                ("example", "hypothesis_explicit_examples"),
+                ("seed", "_hypothesis_internal_use_seed"),
+                ("settings", "_hypothesis_internal_settings_applied"),
+                ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"),
+            ]:
+                if hasattr(item.obj, attribute):
+                    from hypothesis.errors import InvalidArgument
+
+                    raise InvalidArgument(message % (name,))
+            yield
+        else:
+            from hypothesis import HealthCheck, settings
+            from hypothesis.internal.escalation import current_pytest_item
+            from hypothesis.internal.healthcheck import fail_health_check
+            from hypothesis.reporting import with_reporter
+            from hypothesis.statistics import collector, describe_statistics
+
+            # Retrieve the settings for this test from the test object, which
+            # is normally a Hypothesis wrapped_test wrapper. If this doesn't
+            # work, the test object is probably something weird
+            # (e.g a stateful test wrapper), so we skip the function-scoped
+            # fixture check.
+            settings = getattr(item.obj, "_hypothesis_internal_use_settings", None)
+
+            # Check for suspicious use of function-scoped fixtures, but only
+            # if the corresponding health check is not suppressed.
+            if (
+                settings is not None
+                and HealthCheck.function_scoped_fixture
+                not in settings.suppress_health_check
+            ):
+                # Warn about function-scoped fixtures, excluding autouse fixtures because
+                # the advice is probably not actionable and the status quo seems OK...
+                # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail.
+                argnames = None
+                for fx_defs in item._request._fixturemanager.getfixtureinfo(
+                    node=item, func=item.function, cls=None
+                ).name2fixturedefs.values():
+                    if argnames is None:
+                        argnames = frozenset(signature(item.function).parameters)
+                    for fx in fx_defs:
+                        if fx.argname in argnames:
+                            active_fx = item._request._get_active_fixturedef(fx.argname)
+                            if active_fx.scope == "function":
+                                fail_health_check(
+                                    settings,
+                                    _FIXTURE_MSG.format(fx.argname, item.nodeid),
+                                    HealthCheck.function_scoped_fixture,
+                                )
+
+            if item.get_closest_marker("parametrize") is not None:
+                # Give every parametrized test invocation a unique database key
+                key = item.nodeid.encode()
+                item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key
+
+            store = StoringReporter(item.config)
+
+            def note_statistics(stats):
+                stats["nodeid"] = item.nodeid
+                item.hypothesis_statistics = base64.b64encode(
+                    describe_statistics(stats).encode()
+                ).decode()
+
+            with collector.with_value(note_statistics):
+                with with_reporter(store):
+                    with current_pytest_item.with_value(item):
+                        yield
+            if store.results:
+                item.hypothesis_report_information = list(store.results)
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_makereport(item, call):
+        report = (yield).get_result()
+        if hasattr(item, "hypothesis_report_information"):
+            report.sections.append(
+                ("Hypothesis", "\n".join(item.hypothesis_report_information))
+            )
+        if hasattr(item, "hypothesis_statistics") and report.when == "teardown":
+            name = "hypothesis-statistics-" + item.nodeid
+            try:
+                item.config._xml.add_global_property(name, item.hypothesis_statistics)
+            except AttributeError:
+                # --junitxml not passed, or Pytest 4.5 (before add_global_property)
+                # We'll fail xunit2 xml schema checks, upgrade pytest if you care.
+                report.user_properties.append((name, item.hypothesis_statistics))
+            # If there's an HTML report, include our summary stats for each test
+            stats = base64.b64decode(item.hypothesis_statistics.encode()).decode()
+            pytest_html = item.config.pluginmanager.getplugin("html")
+            if pytest_html is not None:  # pragma: no cover
+                report.extra = getattr(report, "extra", []) + [
+                    pytest_html.extras.text(stats, name="Hypothesis stats")
+                ]
+
+    def pytest_terminal_summary(terminalreporter):
+        if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
+            return
+        terminalreporter.section("Hypothesis Statistics")
+
+        def report(properties):
+            for name, value in properties:
+                if name.startswith("hypothesis-statistics-"):
+                    if hasattr(value, "uniobj"):
+                        # Under old versions of pytest, `value` was a `py.xml.raw`
+                        # rather than a string, so we get the (unicode) string off it.
+                        value = value.uniobj
+                    line = base64.b64decode(value.encode()).decode() + "\n\n"
+                    terminalreporter.write_line(line)
+
+        try:
+            global_properties = terminalreporter.config._xml.global_properties
+        except AttributeError:
+            # terminalreporter.stats is a dict, where the empty string appears to
+            # always be the key for a list of _pytest.reports.TestReport objects
+            for test_report in terminalreporter.stats.get("", []):
+                if test_report.when == "teardown":
+                    report(test_report.user_properties)
+        else:
+            report(global_properties)
+
+    def pytest_collection_modifyitems(items):
+        if "hypothesis" not in sys.modules:
+            return
+
+        from hypothesis.internal.detection import is_hypothesis_test
+
+        for item in items:
+            if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj):
+                item.add_marker("hypothesis")
+
+
+def load():
+    """Required for `pluggy` to load a plugin from setuptools entrypoints."""