/
test_statistics.py
130 lines (94 loc) · 3.8 KB
/
test_statistics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# This file is part of Hypothesis, which may be found at
# https://github.com/HypothesisWorks/hypothesis/
#
# Most of this work is copyright (C) 2013-2021 David R. MacIver
# (david@drmaciver.com), but it contains contributions by others. See
# CONTRIBUTING.rst for a full list of people who may hold copyright, and
# consult the git log if you need to determine who owns an individual
# contribution.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.
#
# END HEADER
from distutils.version import LooseVersion
import pytest
from _hypothesis_pytestplugin import PRINT_STATISTICS_OPTION
pytest_plugins = "pytester"
def get_output(testdir, suite, *args):
script = testdir.makepyfile(suite)
result = testdir.runpytest(script, *args)
return "\n".join(result.stdout.lines)
TESTSUITE = """
from hypothesis import HealthCheck, given, settings, assume
from hypothesis.strategies import integers
import time
import warnings
from hypothesis.errors import HypothesisDeprecationWarning
warnings.simplefilter('always', HypothesisDeprecationWarning)
@given(integers())
def test_all_valid(x):
pass
@settings(max_examples=100, suppress_health_check=HealthCheck.all())
@given(integers())
def test_iterations(x):
assume(x == 13)
"""
def test_does_not_run_statistics_by_default(testdir):
out = get_output(testdir, TESTSUITE)
assert "Hypothesis Statistics" not in out
def test_prints_statistics_given_option(testdir):
out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION)
assert "Hypothesis Statistics" in out
assert "max_examples=100" in out
assert "< 10% of examples satisfied assumptions" in out
def test_prints_statistics_given_option_under_xdist(testdir):
out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "-n", "2")
assert "Hypothesis Statistics" in out
assert "max_examples=100" in out
assert "< 10% of examples satisfied assumptions" in out
def test_prints_statistics_given_option_with_junitxml(testdir):
out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "--junit-xml=out.xml")
assert "Hypothesis Statistics" in out
assert "max_examples=100" in out
assert "< 10% of examples satisfied assumptions" in out
@pytest.mark.skipif(LooseVersion(pytest.__version__) < "5.4.0", reason="too old")
def test_prints_statistics_given_option_under_xdist_with_junitxml(testdir):
out = get_output(
testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "-n", "2", "--junit-xml=out.xml"
)
assert "Hypothesis Statistics" in out
assert "max_examples=100" in out
assert "< 10% of examples satisfied assumptions" in out
UNITTEST_TESTSUITE = """
from hypothesis import given
from hypothesis.strategies import integers
from unittest import TestCase
class TestStuff(TestCase):
@given(integers())
def test_all_valid(self, x):
pass
"""
def test_prints_statistics_for_unittest_tests(testdir):
script = testdir.makepyfile(UNITTEST_TESTSUITE)
result = testdir.runpytest(script, PRINT_STATISTICS_OPTION)
out = "\n".join(result.stdout.lines)
assert "Hypothesis Statistics" in out
assert "TestStuff::test_all_valid" in out
assert "max_examples=100" in out
STATEFUL_TESTSUITE = """
from hypothesis.stateful import RuleBasedStateMachine, rule
class Stuff(RuleBasedStateMachine):
@rule()
def step(self):
pass
TestStuff = Stuff.TestCase
"""
def test_prints_statistics_for_stateful_tests(testdir):
script = testdir.makepyfile(STATEFUL_TESTSUITE)
result = testdir.runpytest(script, PRINT_STATISTICS_OPTION)
out = "\n".join(result.stdout.lines)
assert "Hypothesis Statistics" in out
assert "TestStuff::runTest" in out
assert "max_examples=100" in out