From fb1ea3e659e8bcf936032d325cf285cab556f42b Mon Sep 17 00:00:00 2001 From: Marko Ristin Date: Thu, 10 Dec 2020 17:33:50 +0100 Subject: [PATCH] finished with test, started on ghostwrite --- README.rst | 2 +- benchmark.py | 2 +- benchmarks/import_cost/generate.py | 2 +- benchmarks/import_cost/measure.py | 2 +- benchmarks/import_cost/runme.py | 2 +- ...ypothesis.py => pyicontract_hypothesis.py} | 296 ++++++++++++++---- precommit.py | 2 +- setup.py | 2 +- .../with_hypothesis/sample_invalid_module.py | 8 + .../with_hypothesis/sample_module.py | 26 +- .../test_pyicontract_hypothesis.py | 249 ++++++++++++--- 11 files changed, 468 insertions(+), 125 deletions(-) rename icontract/integration/with_hypothesis/{icontract_hypothesis.py => pyicontract_hypothesis.py} (63%) create mode 100644 tests/integration/with_hypothesis/sample_invalid_module.py diff --git a/README.rst b/README.rst index db5af98..37dc72d 100644 --- a/README.rst +++ b/README.rst @@ -566,7 +566,7 @@ Here is some example code: self.assertEqual(123, some_func(15)) if __name__ == '__main__': - unittest.main() + unittest.entry_point() Run this bash command to execute the unit test with slow contracts: diff --git a/benchmark.py b/benchmark.py index 0e6eaab..0aaf09d 100644 --- a/benchmark.py +++ b/benchmark.py @@ -73,7 +73,7 @@ def benchmark_against_others(repo_root: pathlib.Path, overwrite: bool) -> None: def main() -> int: - """"Execute main routine.""" + """"Execute entry_point routine.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--overwrite", help="Overwrites the corresponding section in the Readme.", action='store_true') diff --git a/benchmarks/import_cost/generate.py b/benchmarks/import_cost/generate.py index 2c3d650..663ddff 100755 --- a/benchmarks/import_cost/generate.py +++ b/benchmarks/import_cost/generate.py @@ -58,7 +58,7 @@ def some_func(self) -> None: def main() -> None: - """"Execute the main routine.""" + """"Execute the entry_point routine.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--outdir", help="output directory", default=os.path.dirname(__file__)) args = parser.parse_args() diff --git a/benchmarks/import_cost/measure.py b/benchmarks/import_cost/measure.py index c5b3761..3a8df54 100755 --- a/benchmarks/import_cost/measure.py +++ b/benchmarks/import_cost/measure.py @@ -10,7 +10,7 @@ def main() -> None: - """"Execute the main routine.""" + """"Execute the entry_point routine.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--module", diff --git a/benchmarks/import_cost/runme.py b/benchmarks/import_cost/runme.py index 7a82834..722e58d 100755 --- a/benchmarks/import_cost/runme.py +++ b/benchmarks/import_cost/runme.py @@ -7,7 +7,7 @@ def main() -> None: - """"Execute the main routine.""" + """"Execute the entry_point routine.""" modules = [ "functions_100_with_no_contract", "functions_100_with_1_contract", diff --git a/icontract/integration/with_hypothesis/icontract_hypothesis.py b/icontract/integration/with_hypothesis/pyicontract_hypothesis.py similarity index 63% rename from icontract/integration/with_hypothesis/icontract_hypothesis.py rename to icontract/integration/with_hypothesis/pyicontract_hypothesis.py index 01496a6..bdf7deb 100644 --- a/icontract/integration/with_hypothesis/icontract_hypothesis.py +++ b/icontract/integration/with_hypothesis/pyicontract_hypothesis.py @@ -14,9 +14,9 @@ import sys import tokenize import types -from typing import List, Optional, Tuple, TextIO, Mapping, Any, MutableMapping, Union, Callable, Set +from typing import List, Optional, Tuple, TextIO, Mapping, Any, MutableMapping, Union, Callable, Set, Dict -import asttokens +import hypothesis import icontract @@ -122,9 +122,9 @@ def _parse_general_params(args: argparse.Namespace) -> Tuple[Optional[ParamsGene class ParamsTest: """Represent parameters of the command "test".""" - def __init__(self, path: pathlib.Path, setting: Mapping[str, Any]) -> None: + def __init__(self, path: pathlib.Path, settings: Mapping[str, Any]) -> None: self.path = path - self.setting = setting + self.settings = settings _SETTING_STATEMENT_RE = re.compile(r'^(?P[a-zA-Z_][a-zA-Z_0-9]*)\s*=\s*(?P.*)\s*$') @@ -140,10 +140,10 @@ def _parse_test_params(args: argparse.Namespace) -> Tuple[Optional[ParamsTest], path = pathlib.Path(args.path) - setting = collections.OrderedDict() # type: MutableMapping[str, Any] + settings = collections.OrderedDict() # type: MutableMapping[str, Any] - if args.setting is not None: - for i, statement in enumerate(args.setting): + if args.settings is not None: + for i, statement in enumerate(args.settings): mtch = _SETTING_STATEMENT_RE.match(statement) if not mtch: errors.append("Invalid setting statement {}. Expected statement to match {}, but got: {}".format( @@ -160,19 +160,19 @@ def _parse_test_params(args: argparse.Namespace) -> Tuple[Optional[ParamsTest], errors.append("Failed to parse the value of the setting {}: {}".format(identifier, error)) return None, errors - setting[identifier] = value + settings[identifier] = value if errors: return None, errors - return ParamsTest(path=path, setting=setting), errors + return ParamsTest(path=path, settings=settings), errors class ParamsGhostwrite: """Represent parameters of the command "ghostwrite".""" - def __init__(self, module: str, output: Optional[pathlib.Path], explicit: bool, bare: bool) -> None: - self.module = module + def __init__(self, module_name: str, output: Optional[pathlib.Path], explicit: bool, bare: bool) -> None: + self.module_name = module_name self.output = output self.explicit = explicit self.bare = bare @@ -186,7 +186,7 @@ def _parse_ghostwrite_params(args: argparse.Namespace) -> Tuple[Optional[ParamsG """ output = pathlib.Path(args.output) if args.output != '-' else None - return ParamsGhostwrite(module=args.module, output=output, explicit=args.explicit, bare=args.bare), [] + return ParamsGhostwrite(module_name=args.module, output=output, explicit=args.explicit, bare=args.bare), [] class Params: @@ -242,9 +242,9 @@ def _make_argument_parser() -> argparse.ArgumentParser: test_parser.add_argument("-p", "--path", help="Path to the Python file to test", required=True) test_parser.add_argument( - "--setting", + "--settings", help=("Specify settings for Hypothesis\n\n" - "The settings are separated by ';' and assigned by '='." + "The settings are assigned by '='." "The value of the setting needs to be encoded as JSON.\n\n" "Example: max_examples=500"), nargs="*" @@ -335,7 +335,7 @@ def captured_output(): _DIRECTIVE_RE = re.compile(r'^#\s*pyicontract-hypothesis\s*:\s*(?P[^ \t]*)\s*$') -class Point: +class FunctionPoint: """Represent a testable function.""" @icontract.require(lambda first_row: first_row > 0) @@ -352,13 +352,37 @@ def __init__(self, first_row: int, last_row: int, func: Callable[..., Any]) -> N self.func = func -def _select_points( +def _overlap(first: int, last: int, another_first: int, another_last: int) -> bool: + """ + Return True if the two intervals overlap. + + >>> not any([ + ... _overlap(1, 1, 2, 2), + ... _overlap(2, 2, 1, 1) + ... ]) + True + + >>> all([ + ... _overlap(1, 1, 1, 1), + ... _overlap(1, 5, 1, 1), + ... _overlap(1, 1, 1, 5), + ... _overlap(1, 3, 2, 5), + ... _overlap(2, 5, 1, 3), + ... _overlap(1, 5, 2, 3), + ... _overlap(2, 3, 1, 5), + ... ]) + True + """ + return min(last, another_last) - max(first, another_first) >= 0 + + +def _select_function_points( source_code: str, mod: types.ModuleType, include: List[Union[LineRange, re.Pattern]], exclude: List[Union[LineRange, re.Pattern]] -) -> Tuple[List[Point], List[str]]: - points = [] # type: List[Point] +) -> Tuple[List[FunctionPoint], List[str]]: + included = [] # type: List[FunctionPoint] errors = [] # type: List[str] for key in dir(mod): @@ -366,39 +390,11 @@ def _select_points( if inspect.isfunction(value): func = value # type: Callable[..., Any] source_lines, srow = inspect.getsourcelines(func) - point = Point(first_row=srow, last_row=srow + len(source_lines) - 1, func=func) - points.append(point) - - ## - # Exclude functions which have the disable directive in the body - ## - - included = points - - # TODO: test invalid value different from disable-once - - excluded = set() # type: Set[Point] - for point in points: - reader = io.BytesIO(inspect.getsource(point.func).encode('utf-8')) - for toktype, _, (first_row, _), _, line in tokenize.tokenize(reader.readline): - if toktype == tokenize.COMMENT: - mtch = _DIRECTIVE_RE.match(line.strip()) - if mtch: - value = mtch.group('value') - - if value != 'disable-once': - errors.append( - ("Unexpected directive within a function {} on line {}. " - "Only '# pyicontract-hypothesis: disable-once' expected, " - "but got: {}").format(point.func.__name__, first_row + point.first_row - 1, line.strip())) - continue + point = FunctionPoint(first_row=srow, last_row=srow + len(source_lines) - 1, func=func) + included.append(point) - excluded.add(point) - - if errors: - return [], errors - - included = [point for point in points if point not in excluded] + # The built-in dir() gives us an unsorted directory. + included = sorted(included, key=lambda point: point.first_row) ## # Add ranges of lines given by comment directives to the ``exclude`` @@ -406,8 +402,6 @@ def _select_points( extended_exclude = exclude[:] - # TODO: test invalid value different from enable/disable - range_start = None # type: Optional[int] reader = io.BytesIO(source_code.encode('utf-8')) for toktype, _, (first_row, _), _, line in tokenize.tokenize(reader.readline): @@ -437,18 +431,98 @@ def _select_points( else: raise AssertionError("Unexpected value: {}".format(json.dumps(value))) + exclude = extended_exclude + if errors: return [], errors ## # Remove ``included`` which do not match ``include`` ## - # TODO: two-iterator sweep + + if len(include) > 0: + incl_line_ranges = [incl for incl in include if isinstance(incl, LineRange)] + if len(incl_line_ranges) > 100: + # yapf: disable + print( + ("There are much more --include items then expected: {0}. " + "Please consider filing an issue by visiting this link: " + "https://github.com/Parquery/icontract/issues/new" + "?title=Use+interval+tree" + "&body=We+had+{0}+include+line+ranges+in+pyicontract-hypothesis." + ).format(len(incl_line_ranges))) + # yapf: enable + + if len(incl_line_ranges) > 0: + filtered_included = [] # type: List[FunctionPoint] + for point in included: + # yapf: disable + overlaps_include = any( + _overlap(first=line_range.first, last=line_range.last, + another_first=point.first_row, another_last=point.last_row) + for line_range in incl_line_ranges) + # yapf: enable + + if overlaps_include: + filtered_included.append(point) + + included = filtered_included + + # Match regular expressions + patterns = [incl for incl in include if isinstance(incl, re.Pattern)] + if len(patterns) > 0: + filtered_included = [] + for pattern in patterns: + for point in included: + if pattern.match(point.func.__name__): + filtered_included.append(point) + + included = filtered_included + + if len(included) == 0: + return [], [] ## - # Exclude all points in ``included`` if matched in ``extended_exclude`` + # Exclude all points in ``included`` if matched in ``exclude`` ## - # TODO: two-iterator sweep + + if len(exclude) > 0: + excl_line_ranges = [excl for excl in exclude if isinstance(excl, LineRange)] + if len(excl_line_ranges) > 100: + # yapf: disable + print( + ("There are much more --exclude items then expected: {0}. " + "Please consider filing an issue by visiting this link: " + "https://github.com/Parquery/icontract/issues/new" + "?title=Use+interval+tree" + "&body=We+had+{0}+exclude+line+ranges+in+pyicontract-hypothesis." + ).format(len(excl_line_ranges))) + # yapf: enable + + if len(excl_line_ranges) > 0: + filtered_included = [] + for point in included: + # yapf: disable + overlaps_exclude = any( + _overlap(first=line_range.first, last=line_range.last, + another_first=point.first_row, another_last=point.last_row) + for line_range in excl_line_ranges) + # yapf: enable + + if not overlaps_exclude: + filtered_included.append(point) + + included = filtered_included + + patterns = [excl for excl in exclude if isinstance(excl, re.Pattern)] + if len(patterns) > 0: + filtered_included = [] # type: List[FunctionPoint] + for pattern in patterns: + for point in included: + if not pattern.match(point.func.__name__): + filtered_included.append(point) + + included = filtered_included # TODO: test at point level, manually calling _load_module_from_source_file @@ -476,6 +550,38 @@ def _load_module_from_source_file(path: pathlib.Path) -> Tuple[Optional[types.Mo return mod, [] +def _test_function_point(point: FunctionPoint, settings: Optional[Mapping[str, Any]]) -> List[str]: + """ + Test a single function point. + + Return errors if any. + """ + errors = [] # type: List[str] + + func = point.func # Optimize the look-up + assume_preconditions = icontract.integration.with_hypothesis.make_assume_preconditions(func=func) + + def execute(*args: Tuple[Any, ...], **kwargs: Dict[str, Any]) -> None: + assume_preconditions(*args, **kwargs) + func(*args, **kwargs) + + strategies = icontract.integration.with_hypothesis.infer_strategies(func=func) + + if len(strategies) == 0: + errors.append( + ("No strategies could be inferred for the function: {}. " + "Have you provided type hints for the arguments?").format(func)) + return errors + + wrapped = hypothesis.given(**strategies)(execute) + if settings: + wrapped = hypothesis.settings(**settings)(wrapped) + + wrapped() + + return [] + + def test(general: ParamsGeneral, command: ParamsTest) -> List[str]: """ Test the specified functions. @@ -494,11 +600,31 @@ def test(general: ParamsGeneral, command: ParamsTest) -> List[str]: if errors: return errors - points, errors = _select_points(source_code=source_code, mod=mod, include=general.include, exclude=general.exclude) + points, errors = _select_function_points(source_code=source_code, mod=mod, include=general.include, + exclude=general.exclude) if errors: return errors - print(f"points is {points!r}") # TODO: debug + for point in points: + test_errors = _test_function_point(point=point, settings=command.settings) + errors.extend(test_errors) + + if errors: + return errors + + +def _load_module_with_name(name: str) -> Tuple[Optional[types.ModuleType], List[str]]: + """ + Load the module given its name. + + Example identifier: some.module + """ + try: + mod = importlib.import_module(name=name) + assert isinstance(mod, types.ModuleType) + return mod, [] + except Exception as error: + return None, ["Failed to import the module {}: {}".format(name, error)] def ghostwrite(general: ParamsGeneral, command: ParamsGhostwrite) -> Tuple[str, List[str]]: @@ -507,11 +633,49 @@ def ghostwrite(general: ParamsGeneral, command: ParamsGhostwrite) -> Tuple[str, Return (generated code, errors if any). """ - raise NotImplementedError() - - -def testable_main(argv: List[str], stdout: TextIO, stderr: TextIO) -> int: - """Execute the testable_main routine.""" + mod, errors = _load_module_with_name(command.module_name) + if errors: + return '', errors + + blocks = ["""Test {} with Hypothesis.""".format(command.module_name)] # type: List[str] + # TODO: write imports + # TODO: ghostwrite for the points + # TODO: check if bare + # TODO: refactor to a separate function + + + + return '\n\n'.join(blocks), [] + + # TODO: rem + # ghostwriter_parser = subparsers.add_parser( + # "ghostwrite", help="Ghostwrite the unit test module based on inferred search strategies") + # + # ghostwriter_parser.add_argument("-m", "--module", help="Module to process", required=True) + # + # ghostwriter_parser.add_argument( + # "-o", "--output", + # help="Path to the file where the output should be written. If '-', writes to STDOUT.", + # default="-") + # + # ghostwriter_parser.add_argument( + # "--explicit", + # help=("Write the strategies explicitly in the unit test module instead of inferring them at run-time\n\n" + # "This is practical if you want to tune and refine the strategies and " + # "just want to use ghostwriting as a starting point."), + # action='store_true') + # + # ghostwriter_parser.add_argument( + # "--bare", + # help=("Print only the body of the tests and omit header/footer " + # "(such as TestCase class or import statements).\n\n" + # "This is useful when you only want to inspect a single test or " + # "include a single test function in a custom test suite."), + # action='store_true') + + +def run(argv: List[str], stdout: TextIO, stderr: TextIO) -> int: + """Execute the run routine.""" parser = _make_argument_parser() args, out, err = _parse_args(parser=parser, argv=argv) print(out, file=stdout) @@ -541,10 +705,10 @@ def testable_main(argv: List[str], stdout: TextIO, stderr: TextIO) -> int: return 0 -def main() -> int: - """Wrap the main routine wit default arguments.""" - return testable_main(argv=sys.argv[1:], stdout=sys.stdout, stderr=sys.stderr) +def entry_point() -> int: + """Wrap the entry_point routine wit default arguments.""" + return run(argv=sys.argv[1:], stdout=sys.stdout, stderr=sys.stderr) if __name__ == "__main__": - sys.exit(main()) + sys.exit(entry_point()) diff --git a/precommit.py b/precommit.py index 2e9fc99..d541857 100755 --- a/precommit.py +++ b/precommit.py @@ -8,7 +8,7 @@ def main() -> int: - """"Execute main routine.""" + """"Execute entry_point routine.""" parser = argparse.ArgumentParser() parser.add_argument( "--overwrite", diff --git a/setup.py b/setup.py index 6c5780b..a11ba14 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ package_data={"icontract": ["py.typed"]}, entry_points={ 'console_scripts': [ - 'pyicontract-hypothesis = icontract.integration.with_hypothesis.icontract_hypothesis:main' + 'pyicontract-hypothesis = icontract.integration.with_hypothesis.icontract_hypothesis:entry_point' ], } ) diff --git a/tests/integration/with_hypothesis/sample_invalid_module.py b/tests/integration/with_hypothesis/sample_invalid_module.py new file mode 100644 index 0000000..b825793 --- /dev/null +++ b/tests/integration/with_hypothesis/sample_invalid_module.py @@ -0,0 +1,8 @@ +"""Provide an invalid module for testing pyicontract-hypothesis.""" + +import icontract + +@icontract.require(lambda x: x > 0) +def some_func(x: int) -> None: + # pyicontract-hypothesis: disable-once + pass diff --git a/tests/integration/with_hypothesis/sample_module.py b/tests/integration/with_hypothesis/sample_module.py index 7b6381d..33f9475 100644 --- a/tests/integration/with_hypothesis/sample_module.py +++ b/tests/integration/with_hypothesis/sample_module.py @@ -1,29 +1,29 @@ -"""A sample module meant for testing pyicontract-hypothesis.""" +"""Provide a valid module for testing pyicontract-hypothesis.""" import icontract +SOME_FUNC_CALLS = 0 @icontract.require(lambda x: x > 0) -def testable_some_func(x: int) -> None: - pass +def some_func(x: int) -> None: + global SOME_FUNC_CALLS + SOME_FUNC_CALLS += 1 +ANOTHER_FUNC_CALLS = 0 -def untestable_some_func(x: int) -> None: - # We need more lines so that we can test the overlaps easily. - pass - pass - pass +@icontract.require(lambda x: x > 0) +def another_func(x: int) -> None: + global ANOTHER_FUNC_CALLS + ANOTHER_FUNC_CALLS+=1 -def untestable_another_func(x: int) -> None: - # pyicontract-hypothesis: disable-for-this-function - pass # pyicontract-hypothesis: disable -def untestable_yet_another_func(x: int) -> None: +def expected_to_be_ignored(x: int) -> None: pass -# pyicontract-hypothesis: enable +# pyicontract-hypothesis: enable + class A: @icontract.require(lambda x: x > 0) def __init__(self, x: int) -> None: diff --git a/tests/integration/with_hypothesis/test_pyicontract_hypothesis.py b/tests/integration/with_hypothesis/test_pyicontract_hypothesis.py index 47f2870..b029022 100644 --- a/tests/integration/with_hypothesis/test_pyicontract_hypothesis.py +++ b/tests/integration/with_hypothesis/test_pyicontract_hypothesis.py @@ -5,62 +5,63 @@ import os import pathlib import re -import textwrap import unittest import uuid -import icontract.integration.with_hypothesis.icontract_hypothesis as icontract_hypothesis +import icontract.integration.with_hypothesis.pyicontract_hypothesis as pyicontract_hypothesis +# TODO: split these tests in separate modules, test_common.py, test_test.py and test_ghostwrite.py + class TestLineRangeRe(unittest.TestCase): def test_only_first(self) -> None: - mtch = icontract_hypothesis._LINE_RANGE_RE.match(' 123 ') + mtch = pyicontract_hypothesis._LINE_RANGE_RE.match(' 123 ') assert mtch is not None self.assertEqual('123', mtch.group('first')) self.assertIsNone(mtch.group('last'), "Unexpected last group: {}".format(mtch.group('last'))) def test_first_and_last(self) -> None: - mtch = icontract_hypothesis._LINE_RANGE_RE.match(' 123 - 435 ') + mtch = pyicontract_hypothesis._LINE_RANGE_RE.match(' 123 - 435 ') assert mtch is not None self.assertEqual('123', mtch.group('first')) self.assertEqual('435', mtch.group('last')) def test_no_match(self) -> None: - mtch = icontract_hypothesis._LINE_RANGE_RE.match('123aa') + mtch = pyicontract_hypothesis._LINE_RANGE_RE.match('123aa') assert mtch is None, "Expected no match, but got: {}".format(mtch) class TestParsingOfPointSpecs(unittest.TestCase): def test_single_line(self) -> None: text = '123' - point_spec, errors = icontract_hypothesis._parse_point_spec(text=text) + point_spec, errors = pyicontract_hypothesis._parse_point_spec(text=text) self.assertListEqual([], errors) - assert isinstance(point_spec, icontract_hypothesis.LineRange) + assert isinstance(point_spec, pyicontract_hypothesis.LineRange) self.assertEqual(123, point_spec.first) self.assertEqual(123, point_spec.last) def test_line_range(self) -> None: text = '123-345' - point_spec, errors = icontract_hypothesis._parse_point_spec(text=text) + point_spec, errors = pyicontract_hypothesis._parse_point_spec(text=text) self.assertListEqual([], errors) - assert isinstance(point_spec, icontract_hypothesis.LineRange) + assert isinstance(point_spec, pyicontract_hypothesis.LineRange) self.assertEqual(123, point_spec.first) self.assertEqual(345, point_spec.last) def test_invalid_line_range(self) -> None: text = '345-123' - point_spec, errors = icontract_hypothesis._parse_point_spec(text=text) + point_spec, errors = pyicontract_hypothesis._parse_point_spec(text=text) assert point_spec is None self.assertListEqual(['Unexpected line range (last < first): 345-123'], errors) def test_pattern(self) -> None: text = r'^do_.*$' - point_spec, errors = icontract_hypothesis._parse_point_spec(text=text) + point_spec, errors = pyicontract_hypothesis._parse_point_spec(text=text) assert isinstance(point_spec, re.Pattern) self.assertListEqual([], errors) @@ -73,7 +74,7 @@ def test_no_command(self) -> None: stdout, stderr = io.StringIO(), io.StringIO() - icontract_hypothesis.testable_main(argv=argv, stdout=stdout, stderr=stderr) + pyicontract_hypothesis.run(argv=argv, stdout=stdout, stderr=stderr) stdout.seek(0) out = stdout.read() @@ -96,21 +97,21 @@ def test_subcommand_test(self) -> None: '--setting', 'suppress_health_check=[2, 3]'] # yapf: enable - parser = icontract_hypothesis._make_argument_parser() - args, out, err = icontract_hypothesis._parse_args(parser=parser, argv=argv) + parser = pyicontract_hypothesis._make_argument_parser() + args, out, err = pyicontract_hypothesis._parse_args(parser=parser, argv=argv) assert args is not None, "Failed to parse argv {!r}: {}".format(argv, err) - general, errs = icontract_hypothesis._parse_general_params(args=args) + general, errs = pyicontract_hypothesis._parse_general_params(args=args) self.assertListEqual([], errs) self.assertListEqual([re.compile(pattern) for pattern in ["include-something"]], general.include) self.assertListEqual([re.compile(pattern) for pattern in ["exclude-something"]], general.exclude) - test, errs = icontract_hypothesis._parse_test_params(args=args) + test, errs = pyicontract_hypothesis._parse_test_params(args=args) self.assertListEqual([], errs) self.assertEqual(pathlib.Path('some_module.py'), test.path) - self.assertDictEqual({"suppress_health_check": [2, 3]}, dict(test.setting)) + self.assertDictEqual({"suppress_health_check": [2, 3]}, dict(test.settings)) def test_subcommand_ghostwrite(self) -> None: # yapf: disable @@ -121,38 +122,185 @@ def test_subcommand_ghostwrite(self) -> None: '--explicit', '--bare'] # yapf: enable - parser = icontract_hypothesis._make_argument_parser() - args, out, err = icontract_hypothesis._parse_args(parser=parser, argv=argv) + parser = pyicontract_hypothesis._make_argument_parser() + args, out, err = pyicontract_hypothesis._parse_args(parser=parser, argv=argv) assert args is not None, "Failed to parse argv {!r}: {}".format(argv, err) - general, errs = icontract_hypothesis._parse_general_params(args=args) + general, errs = pyicontract_hypothesis._parse_general_params(args=args) self.assertListEqual([], errs) self.assertListEqual([re.compile(pattern) for pattern in ["include-something"]], general.include) self.assertListEqual([re.compile(pattern) for pattern in ["exclude-something"]], general.exclude) - ghostwrite, errs = icontract_hypothesis._parse_ghostwrite_params(args=args) + ghostwrite, errs = pyicontract_hypothesis._parse_ghostwrite_params(args=args) self.assertListEqual([], errs) - self.assertEqual('some_module', ghostwrite.module) + self.assertEqual('some_module', ghostwrite.module_name) self.assertTrue(ghostwrite.explicit) self.assertTrue(ghostwrite.bare) -# TODO: implement filtering based on the module -# TODO: include # pyicontract-hypothesis: disable / # pyicntract-hypothesis: enable as statement -# TODO: test filtering +class TestSelectFunctionPoints(unittest.TestCase): + def test_invalid_module(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_invalid_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), mod=mod, include=[], exclude=[]) + + # yapf: disable + self.assertListEqual( + ["Unexpected directive on line 7. " + "Expected '# pyicontract-hypothesis: (disable|enable)', " + "but got: # pyicontract-hypothesis: disable-once"], errors) + # yapf: enable + + def test_no_include_and_no_exclude(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), mod=mod, include=[], exclude=[]) + self.assertListEqual([], errors) + + self.assertListEqual(['some_func', 'another_func'], [point.func.__name__ for point in points]) + + def test_include_line_range(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, + # A single line that overlaps the function should be enough to include it. + include=[pyicontract_hypothesis.LineRange(first=9, last=9)], + exclude=[]) + self.assertListEqual([], errors) + # yapf: enable + + self.assertListEqual(['some_func'], [point.func.__name__ for point in points]) + + def test_include_pattern(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, + include=[re.compile(r'^some_.*$')], + exclude=[]) + self.assertListEqual([], errors) + # yapf: enable + + self.assertListEqual(['some_func'], [point.func.__name__ for point in points]) + + def test_exclude_line_range(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, + include=[], + # A single line that overlaps the function should be enough to exclude it. + exclude=[pyicontract_hypothesis.LineRange(first=9, last=9)]) + self.assertListEqual([], errors) + # yapf: enable + + self.assertListEqual(['another_func'], [point.func.__name__ for point in points]) + + def test_exclude_pattern(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, + include=[], + exclude=[re.compile(r'^some_.*$')]) + self.assertListEqual([], errors) + # yapf: enable + + self.assertListEqual(['another_func'], [point.func.__name__ for point in points]) -class TestOnSampleModule(unittest.TestCase): - def test_test_nonexisting_file(self) -> None: +class TestTest(unittest.TestCase): + def test_default_behavior(self) -> None: + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, include=[], exclude=[]) + self.assertListEqual([], errors) + # yapf: enable + + for point in points: + test_errors = pyicontract_hypothesis._test_function_point(point=point, settings=None) + self.assertListEqual([], test_errors) + + some_func_calls = getattr(mod, 'SOME_FUNC_CALLS') + self.assertEqual(100, some_func_calls) + + another_func_calls = getattr(mod, 'ANOTHER_FUNC_CALLS') + self.assertEqual(100, another_func_calls) + + def test_settings(self) -> None: + settings = {"max_examples": 10} + + path = pathlib.Path(os.path.realpath(__file__)).parent / "sample_module.py" + + mod, errors = pyicontract_hypothesis._load_module_from_source_file(path=path) + self.assertListEqual([], errors) + + # yapf: disable + points, errors = pyicontract_hypothesis._select_function_points( + source_code=path.read_text(), + mod=mod, include=[], exclude=[]) + self.assertListEqual([], errors) + # yapf: enable + + for point in points: + test_errors = pyicontract_hypothesis._test_function_point(point=point, settings=settings) + self.assertListEqual([], test_errors) + + some_func_calls = getattr(mod, 'SOME_FUNC_CALLS') + self.assertEqual(10, some_func_calls) + + another_func_calls = getattr(mod, 'ANOTHER_FUNC_CALLS') + self.assertEqual(10, another_func_calls) + + +class TestTestViaSmoke(unittest.TestCase): + """Perform smoke testing of the "test" command.""" + + def test_nonexisting_file(self) -> None: path = "doesnt-exist.{}".format(uuid.uuid4()) argv = ['test', '--path', path] stdout = io.StringIO() stderr = io.StringIO() - exit_code = icontract_hypothesis.testable_main(argv=argv, stdout=stdout, stderr=stderr) + exit_code = pyicontract_hypothesis.run(argv=argv, stdout=stdout, stderr=stderr) stderr.seek(0) err = stderr.read() @@ -160,28 +308,51 @@ def test_test_nonexisting_file(self) -> None: self.assertEqual("The file to be tested does not exist: {}".format(path), err.strip()) self.assertEqual(exit_code, 1) - def test_test(self) -> None: + def test_common_case(self) -> None: this_dir = pathlib.Path(os.path.realpath(__file__)).parent - # yapf: disable - argv = ['test', - '--path', str(this_dir / "sample_module.py"), - '--include', '^testable_.*$', - '--exclude', '^untestable_.*$' - ] - # yapf: enable + argv = ['test', '--path', str(this_dir / "sample_module.py")] stdout = io.StringIO() stderr = io.StringIO() - exit_code = icontract_hypothesis.testable_main(argv=argv, stdout=stdout, stderr=stderr) + # This is merely a smoke test. + exit_code = pyicontract_hypothesis.run(argv=argv, stdout=stdout, stderr=stderr) - stdout.seek(0) - out = stdout.read() + stderr.seek(0) + err = stderr.read() + self.assertEqual('', err.strip()) + self.assertEqual(exit_code, 0) + + def test_with_settings(self) -> None: + this_dir = pathlib.Path(os.path.realpath(__file__)).parent + + argv = ['test', '--path', str(this_dir / "sample_module.py"), '--settings', 'max_examples=5'] + + stdout = io.StringIO() + stderr = io.StringIO() + + # This is merely a smoke test. + exit_code = pyicontract_hypothesis.run(argv=argv, stdout=stdout, stderr=stderr) stderr.seek(0) err = stderr.read() + self.assertEqual('', err.strip()) + self.assertEqual(exit_code, 0) + + def test_with_include_exclude(self) -> None: + this_dir = pathlib.Path(os.path.realpath(__file__)).parent + + argv = ['test', '--path', str(this_dir / "sample_module.py"), '--include', '.*_func', '--exclude', 'some.*'] + stdout = io.StringIO() + stderr = io.StringIO() + + # This is merely a smoke test. + exit_code = pyicontract_hypothesis.run(argv=argv, stdout=stdout, stderr=stderr) + + stderr.seek(0) + err = stderr.read() self.assertEqual('', err.strip()) self.assertEqual(exit_code, 0)