From a664d40bd1223db5831a2c9495a302029fc66c34 Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Tue, 4 Feb 2020 14:38:18 +0100 Subject: [PATCH] Reverse / fix meaning of "+/-" in error diffs The convention is "assert result is expected". Pytest's error diffs now reflect this. "-" means that sth. expected is missing in the result and "+" means that there are unexpected extras in the result. Fixes: #3333 --- doc/en/example/reportingdemo.rst | 16 +- src/_pytest/assertion/__init__.py | 2 +- src/_pytest/assertion/util.py | 228 +++++++++++++------------- testing/acceptance_test.py | 4 +- testing/test_assertion.py | 134 ++++++++-------- testing/test_error_diffs.py | 255 ++++++++++++++++++++++++++++++ 6 files changed, 453 insertions(+), 186 deletions(-) create mode 100644 testing/test_error_diffs.py diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 1c06782f631..1ab0f9c828b 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -81,8 +81,8 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_eq_text(self): > assert "spam" == "eggs" E AssertionError: assert 'spam' == 'eggs' - E - spam - E + eggs + E - eggs + E + spam failure_demo.py:45: AssertionError _____________ TestSpecialisedExplanations.test_eq_similar_text _____________ @@ -92,9 +92,9 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_eq_similar_text(self): > assert "foo 1 bar" == "foo 2 bar" E AssertionError: assert 'foo 1 bar' == 'foo 2 bar' - E - foo 1 bar + E - foo 2 bar E ? ^ - E + foo 2 bar + E + foo 1 bar E ? ^ failure_demo.py:48: AssertionError @@ -106,8 +106,8 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert "foo\nspam\nbar" == "foo\neggs\nbar" E AssertionError: assert 'foo\nspam\nbar' == 'foo\neggs\nbar' E foo - E - spam - E + eggs + E - eggs + E + spam E bar failure_demo.py:51: AssertionError @@ -122,9 +122,9 @@ Here is a nice run of several failures and how ``pytest`` presents things: E AssertionError: assert '111111111111...2222222222222' == '111111111111...2222222222222' E Skipping 90 identical leading characters in diff, use -v to show E Skipping 91 identical trailing characters in diff, use -v to show - E - 1111111111a222222222 + E - 1111111111b222222222 E ? ^ - E + 1111111111b222222222 + E + 1111111111a222222222 E ? ^ failure_demo.py:56: AssertionError diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index a060723a76b..504d4e8538b 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -168,4 +168,4 @@ def pytest_sessionfinish(session): def pytest_assertrepr_compare(config, op, left, right): - return util.assertrepr_compare(config=config, op=op, left=left, right=right) + return util.assertrepr_compare(config=config, op=op, result=left, expected=right) diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 67f8d46185e..0266d2f4342 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -128,49 +128,53 @@ def isiterable(obj: Any) -> bool: return False -def assertrepr_compare(config, op: str, left: Any, right: Any) -> Optional[List[str]]: +def assertrepr_compare( + config, op: str, result: Any, expected: Any +) -> Optional[List[str]]: """Return specialised explanations for some operators/operands""" verbose = config.getoption("verbose") if verbose > 1: - left_repr = safeformat(left) - right_repr = safeformat(right) + result_repr = safeformat(result) + expected_repr = safeformat(expected) else: # XXX: "15 chars indentation" is wrong # ("E AssertionError: assert "); should use term width. maxsize = ( 80 - 15 - len(op) - 2 ) // 2 # 15 chars indentation, 1 space around op - left_repr = saferepr(left, maxsize=maxsize) - right_repr = saferepr(right, maxsize=maxsize) + result_repr = saferepr(result, maxsize=maxsize) + expected_repr = saferepr(expected, maxsize=maxsize) - summary = "{} {} {}".format(left_repr, op, right_repr) + summary = "{} {} {}".format(result_repr, op, expected_repr) explanation = None try: if op == "==": - if istext(left) and istext(right): - explanation = _diff_text(left, right, verbose) + if istext(result) and istext(expected): + explanation = _diff_text(result, expected, verbose) else: - if issequence(left) and issequence(right): - explanation = _compare_eq_sequence(left, right, verbose) - elif isset(left) and isset(right): - explanation = _compare_eq_set(left, right, verbose) - elif isdict(left) and isdict(right): - explanation = _compare_eq_dict(left, right, verbose) - elif type(left) == type(right) and (isdatacls(left) or isattrs(left)): + if issequence(result) and issequence(expected): + explanation = _compare_eq_sequence(result, expected, verbose) + elif isset(result) and isset(expected): + explanation = _compare_eq_set(result, expected, verbose) + elif isdict(result) and isdict(expected): + explanation = _compare_eq_dict(result, expected, verbose) + elif type(result) == type(expected) and ( + isdatacls(result) or isattrs(result) + ): type_fn = (isdatacls, isattrs) - explanation = _compare_eq_cls(left, right, verbose, type_fn) + explanation = _compare_eq_cls(result, expected, verbose, type_fn) elif verbose > 0: - explanation = _compare_eq_verbose(left, right) - if isiterable(left) and isiterable(right): - expl = _compare_eq_iterable(left, right, verbose) + explanation = _compare_eq_verbose(result, expected) + if isiterable(result) and isiterable(expected): + expl = _compare_eq_iterable(result, expected, verbose) if explanation is not None: explanation.extend(expl) else: explanation = expl elif op == "not in": - if istext(left) and istext(right): - explanation = _notin_text(left, right, verbose) + if istext(result) and istext(expected): + explanation = _notin_text(result, expected, verbose) except outcomes.Exit: raise except Exception: @@ -186,7 +190,7 @@ def assertrepr_compare(config, op: str, left: Any, right: Any) -> Optional[List[ return [summary] + explanation -def _diff_text(left: str, right: str, verbose: int = 0) -> List[str]: +def _diff_text(result: str, expected: str, verbose: int = 0) -> List[str]: """Return the explanation for the diff between text. Unless --verbose is used this will skip leading and trailing @@ -197,20 +201,20 @@ def _diff_text(left: str, right: str, verbose: int = 0) -> List[str]: explanation = [] # type: List[str] if verbose < 1: - i = 0 # just in case left or right has zero length - for i in range(min(len(left), len(right))): - if left[i] != right[i]: + i = 0 # just in case "result" or "expected" has zero length + for i in range(min(len(result), len(expected))): + if result[i] != expected[i]: break if i > 42: i -= 10 # Provide some context explanation = [ "Skipping %s identical leading characters in diff, use -v to show" % i ] - left = left[i:] - right = right[i:] - if len(left) == len(right): - for i in range(len(left)): - if left[-i] != right[-i]: + result = result[i:] + expected = expected[i:] + if len(result) == len(expected): + for i in range(len(result)): + if result[-i] != expected[-i]: break if i > 42: i -= 10 # Provide some context @@ -218,28 +222,30 @@ def _diff_text(left: str, right: str, verbose: int = 0) -> List[str]: "Skipping {} identical trailing " "characters in diff, use -v to show".format(i) ] - left = left[:-i] - right = right[:-i] + result = result[:-i] + expected = expected[:-i] keepends = True - if left.isspace() or right.isspace(): - left = repr(str(left)) - right = repr(str(right)) + if result.isspace() or expected.isspace(): + result = repr(str(result)) + expected = repr(str(expected)) explanation += ["Strings contain only whitespace, escaping them using repr()"] + # "expected" is the base against which we compare "result", + # see https://github.com/pytest-dev/pytest/issues/3333 explanation += [ line.strip("\n") - for line in ndiff(left.splitlines(keepends), right.splitlines(keepends)) + for line in ndiff(expected.splitlines(keepends), result.splitlines(keepends)) ] return explanation -def _compare_eq_verbose(left: Any, right: Any) -> List[str]: +def _compare_eq_verbose(result: Any, expected: Any) -> List[str]: keepends = True - left_lines = repr(left).splitlines(keepends) - right_lines = repr(right).splitlines(keepends) + result_lines = repr(result).splitlines(keepends) + expected_lines = repr(expected).splitlines(keepends) explanation = [] # type: List[str] - explanation += ["-" + line for line in left_lines] - explanation += ["+" + line for line in right_lines] + explanation += ["+" + line for line in result_lines] + explanation += ["-" + line for line in expected_lines] return explanation @@ -257,43 +263,45 @@ def _surrounding_parens_on_own_lines(lines: List[str]) -> None: def _compare_eq_iterable( - left: Iterable[Any], right: Iterable[Any], verbose: int = 0 + result: Iterable[Any], expected: Iterable[Any], verbose: int = 0 ) -> List[str]: if not verbose: return ["Use -v to get the full diff"] # dynamic import to speedup pytest import difflib - left_formatting = pprint.pformat(left).splitlines() - right_formatting = pprint.pformat(right).splitlines() + result_formatting = pprint.pformat(result).splitlines() + expected_formatting = pprint.pformat(expected).splitlines() # Re-format for different output lengths. - lines_left = len(left_formatting) - lines_right = len(right_formatting) - if lines_left != lines_right: - left_formatting = _pformat_dispatch(left).splitlines() - right_formatting = _pformat_dispatch(right).splitlines() + lines_result = len(result_formatting) + lines_expected = len(expected_formatting) + if lines_result != lines_expected: + result_formatting = _pformat_dispatch(result).splitlines() + expected_formatting = _pformat_dispatch(expected).splitlines() - if lines_left > 1 or lines_right > 1: - _surrounding_parens_on_own_lines(left_formatting) - _surrounding_parens_on_own_lines(right_formatting) + if lines_result > 1 or lines_expected > 1: + _surrounding_parens_on_own_lines(result_formatting) + _surrounding_parens_on_own_lines(expected_formatting) explanation = ["Full diff:"] + # "expected" is the base against which we compare "result", + # see https://github.com/pytest-dev/pytest/issues/3333 explanation.extend( - line.rstrip() for line in difflib.ndiff(left_formatting, right_formatting) + line.rstrip() for line in difflib.ndiff(expected_formatting, result_formatting) ) return explanation def _compare_eq_sequence( - left: Sequence[Any], right: Sequence[Any], verbose: int = 0 + result: Sequence[Any], expected: Sequence[Any], verbose: int = 0 ) -> List[str]: - comparing_bytes = isinstance(left, bytes) and isinstance(right, bytes) + comparing_bytes = isinstance(result, bytes) and isinstance(expected, bytes) explanation = [] # type: List[str] - len_left = len(left) - len_right = len(right) - for i in range(min(len_left, len_right)): - if left[i] != right[i]: + len_result = len(result) + len_expected = len(expected) + for i in range(min(len_result, len_expected)): + if result[i] != expected[i]: if comparing_bytes: # when comparing bytes, we want to see their ascii representation # instead of their numeric values (#5260) @@ -303,31 +311,32 @@ def _compare_eq_sequence( # 102 # >>> s[0:1] # b'f' - left_value = left[i : i + 1] - right_value = right[i : i + 1] + result_value = result[i : i + 1] + expected_value = expected[i : i + 1] else: - left_value = left[i] - right_value = right[i] + result_value = result[i] + expected_value = expected[i] explanation += [ - "At index {} diff: {!r} != {!r}".format(i, left_value, right_value) + "At index {} diff: {!r} != {!r}".format(i, result_value, expected_value) ] break if comparing_bytes: - # when comparing bytes, it doesn't help to show the "sides contain one or more items" - # longer explanation, so skip it + # when comparing bytes, it doesn't help to show the "sides contain one or more + # items" longer explanation, so skip it + return explanation - len_diff = len_left - len_right + len_diff = len_result - len_expected if len_diff: if len_diff > 0: - dir_with_more = "Left" - extra = saferepr(left[len_right]) + dir_with_more = "Result" + extra = saferepr(result[len_expected]) else: len_diff = 0 - len_diff - dir_with_more = "Right" - extra = saferepr(right[len_left]) + dir_with_more = "Expected" + extra = saferepr(expected[len_result]) if len_diff == 1: explanation += [ @@ -342,75 +351,77 @@ def _compare_eq_sequence( def _compare_eq_set( - left: AbstractSet[Any], right: AbstractSet[Any], verbose: int = 0 + result: AbstractSet[Any], expected: AbstractSet[Any], verbose: int = 0 ) -> List[str]: explanation = [] - diff_left = left - right - diff_right = right - left - if diff_left: - explanation.append("Extra items in the left set:") - for item in diff_left: + diff_result = result - expected + diff_expected = expected - result + if diff_result: + explanation.append("Extra items in the result set:") + for item in diff_result: explanation.append(saferepr(item)) - if diff_right: - explanation.append("Extra items in the right set:") - for item in diff_right: + if diff_expected: + explanation.append("Extra items in the expected set:") + for item in diff_expected: explanation.append(saferepr(item)) return explanation def _compare_eq_dict( - left: Mapping[Any, Any], right: Mapping[Any, Any], verbose: int = 0 + result: Mapping[Any, Any], expected: Mapping[Any, Any], verbose: int = 0 ) -> List[str]: explanation = [] # type: List[str] - set_left = set(left) - set_right = set(right) - common = set_left.intersection(set_right) - same = {k: left[k] for k in common if left[k] == right[k]} + set_result = set(result) + set_expected = set(expected) + common = set_result.intersection(set_expected) + same = {k: result[k] for k in common if result[k] == expected[k]} if same and verbose < 2: explanation += ["Omitting %s identical items, use -vv to show" % len(same)] elif same: explanation += ["Common items:"] explanation += pprint.pformat(same).splitlines() - diff = {k for k in common if left[k] != right[k]} + diff = {k for k in common if result[k] != expected[k]} if diff: explanation += ["Differing items:"] for k in diff: - explanation += [saferepr({k: left[k]}) + " != " + saferepr({k: right[k]})] - extra_left = set_left - set_right - len_extra_left = len(extra_left) - if len_extra_left: + explanation += [ + saferepr({k: result[k]}) + " != " + saferepr({k: expected[k]}) + ] + extra_result = set_result - set_expected + len_extra_result = len(extra_result) + if len_extra_result: explanation.append( - "Left contains %d more item%s:" - % (len_extra_left, "" if len_extra_left == 1 else "s") + "Result contains %d more item%s:" + % (len_extra_result, "" if len_extra_result == 1 else "s") ) explanation.extend( - pprint.pformat({k: left[k] for k in extra_left}).splitlines() + pprint.pformat({k: result[k] for k in extra_result}).splitlines() ) - extra_right = set_right - set_left - len_extra_right = len(extra_right) - if len_extra_right: + extra_expected = set_expected - set_result + len_extra_expected = len(extra_expected) + if len_extra_expected: explanation.append( - "Right contains %d more item%s:" - % (len_extra_right, "" if len_extra_right == 1 else "s") + "Expected contains %d more item%s:" + % (len_extra_expected, "" if len_extra_expected == 1 else "s") ) explanation.extend( - pprint.pformat({k: right[k] for k in extra_right}).splitlines() + pprint.pformat({k: expected[k] for k in extra_expected}).splitlines() ) return explanation def _compare_eq_cls( - left: Any, - right: Any, + result: Any, + expected: Any, verbose: int, type_fns: Tuple[Callable[[Any], bool], Callable[[Any], bool]], ) -> List[str]: isdatacls, isattrs = type_fns - if isdatacls(left): - all_fields = left.__dataclass_fields__ + if isdatacls(result): + all_fields = result.__dataclass_fields__ fields_to_check = [field for field, info in all_fields.items() if info.compare] - elif isattrs(left): - all_fields = left.__attrs_attrs__ + elif isattrs(result): + all_fields = result.__attrs_attrs__ fields_to_check = [ field.name for field in all_fields if getattr(field, ATTRS_EQ_FIELD) ] @@ -418,7 +429,7 @@ def _compare_eq_cls( same = [] diff = [] for field in fields_to_check: - if getattr(left, field) == getattr(right, field): + if getattr(result, field) == getattr(expected, field): same.append(field) else: diff.append(field) @@ -433,7 +444,8 @@ def _compare_eq_cls( explanation += ["Differing attributes:"] for field in diff: explanation += [ - ("%s: %r != %r") % (field, getattr(left, field), getattr(right, field)) + ("%s: %r != %r") + % (field, getattr(result, field), getattr(expected, field)) ] return explanation @@ -443,7 +455,7 @@ def _notin_text(term: str, text: str, verbose: int = 0) -> List[str]: head = text[:index] tail = text[index + len(term) :] correct_text = head + tail - diff = _diff_text(correct_text, text, verbose) + diff = _diff_text(text, correct_text, verbose) newdiff = ["%s is contained here:" % saferepr(term, maxsize=42)] for line in diff: if line.startswith("Skipping"): diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 68e8a97f8f3..20b370055ba 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1277,8 +1277,8 @@ def test(): " def check():", "> assert 1 == 2", "E assert 1 == 2", - "E -1", - "E +2", + "E +1", + "E -2", "", "pdb.py:2: AssertionError", "*= 1 failed in *", diff --git a/testing/test_assertion.py b/testing/test_assertion.py index e975a3fea2b..f85592ce11a 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -316,8 +316,8 @@ def test_summary(self): def test_text_diff(self): diff = callequal("spam", "eggs")[1:] - assert "- spam" in diff - assert "+ eggs" in diff + assert "- eggs" in diff + assert "+ spam" in diff def test_text_skipping(self): lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs") @@ -327,15 +327,15 @@ def test_text_skipping(self): def test_text_skipping_verbose(self): lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs", verbose=1) - assert "- " + "a" * 50 + "spam" in lines - assert "+ " + "a" * 50 + "eggs" in lines + assert "- " + "a" * 50 + "eggs" in lines + assert "+ " + "a" * 50 + "spam" in lines def test_multiline_text_diff(self): left = "foo\nspam\nbar" right = "foo\neggs\nbar" diff = callequal(left, right) - assert "- spam" in diff - assert "+ eggs" in diff + assert "- eggs" in diff + assert "+ spam" in diff def test_bytes_diff_normal(self): """Check special handling for bytes diff (#5260)""" @@ -354,8 +354,8 @@ def test_bytes_diff_verbose(self): "b'spam' == b'eggs'", "At index 0 diff: b's' != b'e'", "Full diff:", - "- b'spam'", - "+ b'eggs'", + "- b'eggs'", + "+ b'spam'", ] def test_list(self): @@ -370,9 +370,9 @@ def test_list(self): [0, 2], """ Full diff: - - [0, 1] + - [0, 2] ? ^ - + [0, 2] + + [0, 1] ? ^ """, id="lists", @@ -382,9 +382,9 @@ def test_list(self): {0: 2}, """ Full diff: - - {0: 1} + - {0: 2} ? ^ - + {0: 2} + + {0: 1} ? ^ """, id="dicts", @@ -394,9 +394,9 @@ def test_list(self): {0, 2}, """ Full diff: - - {0, 1} + - {0, 2} ? ^ - + {0, 2} + + {0, 1} ? ^ """, id="sets", @@ -427,26 +427,26 @@ def test_list_wrap_for_multiple_lines(self): diff = callequal(l1, l2, verbose=True) assert diff == [ "['a', 'b', 'c'] == ['a', 'b', 'c...dddddddddddd']", - "Right contains one more item: '" + long_d + "'", + "Expected contains one more item: '" + long_d + "'", "Full diff:", " [", " 'a',", " 'b',", " 'c',", - "+ '" + long_d + "',", + "- '" + long_d + "',", " ]", ] diff = callequal(l2, l1, verbose=True) assert diff == [ "['a', 'b', 'c...dddddddddddd'] == ['a', 'b', 'c']", - "Left contains one more item: '" + long_d + "'", + "Result contains one more item: '" + long_d + "'", "Full diff:", " [", " 'a',", " 'b',", " 'c',", - "- '" + long_d + "',", + "+ '" + long_d + "',", " ]", ] @@ -462,10 +462,10 @@ def test_list_wrap_for_width_rewrap_same_length(self): "At index 0 diff: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' != 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'", "Full diff:", " [", - "- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", + "+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", " 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',", " 'cccccccccccccccccccccccccccccc',", - "+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", + "- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", " ]", ] @@ -477,31 +477,31 @@ def test_list_dont_wrap_strings(self): assert diff == [ "['a', 'aaaaaa...aaaaaaa', ...] == ['should not get wrapped']", "At index 0 diff: 'a' != 'should not get wrapped'", - "Left contains 7 more items, first extra item: 'aaaaaaaaaa'", + "Result contains 7 more items, first extra item: 'aaaaaaaaaa'", "Full diff:", " [", - "+ 'should not get wrapped',", - "- 'a',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", - "- 'aaaaaaaaaa',", + "- 'should not get wrapped',", + "+ 'a',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", " ]", ] def test_dict_wrap(self): - d1 = {"common": 1, "env": {"env1": 1}} - d2 = {"common": 1, "env": {"env1": 1, "env2": 2}} + d1 = {"common": 1, "env": {"env1": 1, "env2": 2}} + d2 = {"common": 1, "env": {"env1": 1}} diff = callequal(d1, d2, verbose=True) assert diff == [ - "{'common': 1,...: {'env1': 1}} == {'common': 1,...1, 'env2': 2}}", + "{'common': 1,...1, 'env2': 2}} == {'common': 1,...: {'env1': 1}}", "Omitting 1 identical items, use -vv to show", "Differing items:", - "{'env': {'env1': 1}} != {'env': {'env1': 1, 'env2': 2}}", + "{'env': {'env1': 1, 'env2': 2}} != {'env': {'env1': 1}}", "Full diff:", "- {'common': 1, 'env': {'env1': 1}}", "+ {'common': 1, 'env': {'env1': 1, 'env2': 2}}", @@ -516,14 +516,14 @@ def test_dict_wrap(self): assert diff == [ "{'env': {'sub... wrapped '}}}} == {'env': {'sub...}}}, 'new': 1}", "Omitting 1 identical items, use -vv to show", - "Right contains 1 more item:", + "Expected contains 1 more item:", "{'new': 1}", "Full diff:", " {", " 'env': {'sub': {'long_a': '" + long_a + "',", " 'sub1': {'long_a': 'substring that gets wrapped substring '", " 'that gets wrapped '}}},", - "+ 'new': 1,", + "- 'new': 1,", " }", ] @@ -556,24 +556,24 @@ def test_dict_different_items(self): lines = callequal({"a": 0}, {"b": 1, "c": 2}, verbose=2) assert lines == [ "{'a': 0} == {'b': 1, 'c': 2}", - "Left contains 1 more item:", + "Result contains 1 more item:", "{'a': 0}", - "Right contains 2 more items:", + "Expected contains 2 more items:", "{'b': 1, 'c': 2}", "Full diff:", - "- {'a': 0}", - "+ {'b': 1, 'c': 2}", + "- {'b': 1, 'c': 2}", + "+ {'a': 0}", ] lines = callequal({"b": 1, "c": 2}, {"a": 0}, verbose=2) assert lines == [ "{'b': 1, 'c': 2} == {'a': 0}", - "Left contains 2 more items:", + "Result contains 2 more items:", "{'b': 1, 'c': 2}", - "Right contains 1 more item:", + "Expected contains 1 more item:", "{'a': 0}", "Full diff:", - "- {'b': 1, 'c': 2}", - "+ {'a': 0}", + "- {'a': 0}", + "+ {'b': 1, 'c': 2}", ] def test_sequence_different_items(self): @@ -581,19 +581,19 @@ def test_sequence_different_items(self): assert lines == [ "(1, 2) == (3, 4, 5)", "At index 0 diff: 1 != 3", - "Right contains one more item: 5", + "Expected contains one more item: 5", "Full diff:", - "- (1, 2)", - "+ (3, 4, 5)", + "- (3, 4, 5)", + "+ (1, 2)", ] lines = callequal((1, 2, 3), (4,), verbose=2) assert lines == [ "(1, 2, 3) == (4,)", "At index 0 diff: 1 != 4", - "Left contains 2 more items, first extra item: 2", + "Result contains 2 more items, first extra item: 2", "Full diff:", - "- (1, 2, 3)", - "+ (4,)", + "- (4,)", + "+ (1, 2, 3)", ] def test_set(self): @@ -654,12 +654,12 @@ def __repr__(self): assert callequal(nums_x, nums_y) is None expl = callequal(nums_x, nums_y, verbose=1) - assert "-" + repr(nums_x) in expl - assert "+" + repr(nums_y) in expl + assert "+" + repr(nums_x) in expl + assert "-" + repr(nums_y) in expl expl = callequal(nums_x, nums_y, verbose=2) - assert "-" + repr(nums_x) in expl - assert "+" + repr(nums_y) in expl + assert "+" + repr(nums_x) in expl + assert "-" + repr(nums_y) in expl def test_list_bad_repr(self): class A: @@ -693,8 +693,8 @@ def test_unicode(self): right = "£" expl = callequal(left, right) assert expl[0] == "'£€' == '£'" - assert expl[1] == "- £€" - assert expl[2] == "+ £" + assert expl[1] == "- £" + assert expl[2] == "+ £€" def test_nonascii_text(self): """ @@ -707,7 +707,7 @@ def __repr__(self): return "\xff" expl = callequal(A(), "1") - assert expl == ["ÿ == '1'", "+ 1"] + assert expl == ["ÿ == '1'", "- 1"] def test_format_nonascii_explanation(self): assert util.format_explanation("λ") @@ -1007,9 +1007,9 @@ def test_many_lines(): # without -vv, truncate the message showing a few diff lines only result.stdout.fnmatch_lines( [ - "*- 1*", - "*- 3*", - "*- 5*", + "*+ 1*", + "*+ 3*", + "*+ 5*", "*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines, ] ) @@ -1062,9 +1062,9 @@ def test_reprcompare_whitespaces(): assert detail == [ r"'\r\n' == '\n'", r"Strings contain only whitespace, escaping them using repr()", - r"- '\r\n'", - r"? --", - r"+ '\n'", + r"- '\n'", + r"+ '\r\n'", + r"? ++", ] @@ -1083,7 +1083,7 @@ def test_hello(): [ "*def test_hello():*", "*assert x == y*", - "*E*Extra items*left*", + "*E*Extra items*result*", "*E*50*", "*= 1 failed in*", ] @@ -1104,9 +1104,9 @@ def test_hello(): [ "*def test_hello():*", "*assert x == y*", - "*E*Extra items*left*", + "*E*Extra items*result*", "*E*'x'*", - "*E*Extra items*right*", + "*E*Extra items*expected*", "*E*'y'*", ] ) @@ -1312,8 +1312,8 @@ def test_diff(): r""" *assert 'asdf' == 'asdf\n' * - asdf + * ? - * + asdf - * ? + """ ) diff --git a/testing/test_error_diffs.py b/testing/test_error_diffs.py new file mode 100644 index 00000000000..a11eb0ed7b1 --- /dev/null +++ b/testing/test_error_diffs.py @@ -0,0 +1,255 @@ +""" +Tests and examples for correct "+/-" usage in error diffs. + +See https://github.com/pytest-dev/pytest/issues/3333 for details. + +""" +import pytest + + +TESTCASES = [ + ( # Compare lists, one item differs + """ + def test_this(): + result = [1, 4, 3] + expected = [1, 2, 3] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 4, 3] == [1, 2, 3] + E At index 1 diff: 4 != 2 + E Full diff: + E - [1, 2, 3] + E ? ^ + E + [1, 4, 3] + E ? ^ + """, + ), + ( # Compare lists, one extra item + """ + def test_this(): + result = [1, 2, 3] + expected = [1, 2] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 2, 3] == [1, 2] + E Result contains one more item: 3 + E Full diff: + E - [1, 2] + E + [1, 2, 3] + E ? +++ + """, + ), + ( # Compare lists, one item missing + """ + def test_this(): + result = [1, 3] + expected = [1, 2, 3] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 3] == [1, 2, 3] + E At index 1 diff: 3 != 2 + E Expected contains one more item: 3 + E Full diff: + E - [1, 2, 3] + E ? --- + E + [1, 3] + """, + ), + ( # Compare tuples + """ + def test_this(): + result = (1, 4, 3) + expected = (1, 2, 3) + assert result == expected + """, + """ + > assert result == expected + E assert (1, 4, 3) == (1, 2, 3) + E At index 1 diff: 4 != 2 + E Full diff: + E - (1, 2, 3) + E ? ^ + E + (1, 4, 3) + E ? ^ + """, + ), + ( # Compare sets + """ + def test_this(): + result = {1, 4, 3} + expected = {1, 2, 3} + assert result == expected + """, + """ + > assert result == expected + E assert {1, 3, 4} == {1, 2, 3} + E Extra items in the result set: + E 4 + E Extra items in the expected set: + E 2 + E Full diff: + E - {1, 2, 3} + E ? ^ ^ + E + {1, 3, 4} + E ? ^ ^ + """, + ), + ( # Compare dicts with differing keys + """ + def test_this(): + result = {1: 'spam', 3: 'eggs'} + expected = {1: 'spam', 2: 'eggs'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 3: 'eggs'} == {1: 'spam', 2: 'eggs'} + E Common items: + E {1: 'spam'} + E Result contains 1 more item: + E {3: 'eggs'} + E Expected contains 1 more item: + E {2: 'eggs'} + E Full diff: + E - {1: 'spam', 2: 'eggs'} + E ? ^ + E + {1: 'spam', 3: 'eggs'} + E ? ^ + """, + ), + ( # Compare dicts with differing values + """ + def test_this(): + result = {1: 'spam', 2: 'eggs'} + expected = {1: 'spam', 2: 'bacon'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 2: 'bacon'} + E Common items: + E {1: 'spam'} + E Differing items: + E {2: 'eggs'} != {2: 'bacon'} + E Full diff: + E - {1: 'spam', 2: 'bacon'} + E ? ^^^^^ + E + {1: 'spam', 2: 'eggs'} + E ? ^^^^ + """, + ), + ( # Compare dicts with differing items + """ + def test_this(): + result = {1: 'spam', 2: 'eggs'} + expected = {1: 'spam', 3: 'bacon'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 3: 'bacon'} + E Common items: + E {1: 'spam'} + E Result contains 1 more item: + E {2: 'eggs'} + E Expected contains 1 more item: + E {3: 'bacon'} + E Full diff: + E - {1: 'spam', 3: 'bacon'} + E ? ^ ^^^^^ + E + {1: 'spam', 2: 'eggs'} + E ? ^ ^^^^ + """, + ), + ( # Compare data classes + """ + from dataclasses import dataclass + + @dataclass + class A: + a: int + b: str + + def test_this(): + result = A(1, 'spam') + expected = A(2, 'spam') + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert A(a=1, b='spam') == A(a=2, b='spam') + E Matching attributes: + E ['b'] + E Differing attributes: + E a: 1 != 2 + """, + ), + ( # Compare attrs classes + """ + import attr + + @attr.s(auto_attribs=True) + class A: + a: int + b: str + + def test_this(): + result = A(1, 'spam') + expected = A(1, 'eggs') + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert A(a=1, b='spam') == A(a=1, b='eggs') + E Matching attributes: + E ['a'] + E Differing attributes: + E b: 'spam' != 'eggs' + """, + ), + ( # Compare strings + """ + def test_this(): + result = "spmaeggs" + expected = "spameggs" + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert 'spmaeggs' == 'spameggs' + E - spameggs + E ? - + E + spmaeggs + E ? + + """, + ), + ( # Test "not in" string + """ + def test_this(): + result = "spam bacon eggs" + assert "bacon" not in result + """, + """ + > assert "bacon" not in result + E AssertionError: assert 'bacon' not in 'spam bacon eggs' + E 'bacon' is contained here: + E spam bacon eggs + E ? +++++ + """, + ), +] + + +@pytest.mark.parametrize("code, expected", TESTCASES) +def test_error_diff(code, expected, testdir): + expected = [l.lstrip() for l in expected.splitlines()] + p = testdir.makepyfile(code) + result = testdir.runpytest(p, "-vv") + result.stdout.fnmatch_lines(expected) + assert result.ret == 1