From eea5219eaa17a56c2bb5f0f35a1571d93ba6ff4d Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 9 Jan 2023 16:53:49 -0800 Subject: [PATCH 1/5] Initial implementation of wrapping multiple context managers. --- src/black/linegen.py | 80 ++++++++++++++----- src/black/mode.py | 5 ++ .../preview_38/multiple_context_managers.py | 38 +++++++++ .../wrap_multiple_context_managers.py | 59 ++++++++++++++ tests/test_format.py | 7 ++ 5 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 tests/data/preview_38/multiple_context_managers.py create mode 100644 tests/data/preview_39/wrap_multiple_context_managers.py diff --git a/src/black/linegen.py b/src/black/linegen.py index 4da75b28235..c3e8b46a5b9 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -22,7 +22,7 @@ is_line_short_enough, line_to_string, ) -from black.mode import Feature, Mode, Preview +from black.mode import Feature, Mode, Preview, supports_feature from black.nodes import ( ASSIGNMENTS, BRACKETS, @@ -190,7 +190,7 @@ def visit_stmt( `parens` holds a set of string leaf values immediately after which invisible parens should be put. """ - normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview) + normalize_invisible_parens(node, parens_after=parens, mode=self.mode) for child in node.children: if is_name_token(child) and child.value in keywords: yield from self.line() @@ -243,7 +243,7 @@ def visit_funcdef(self, node: Node) -> Iterator[Line]: def visit_match_case(self, node: Node) -> Iterator[Line]: """Visit either a match or case statement.""" - normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview) + normalize_invisible_parens(node, parens_after=set(), mode=self.mode) yield from self.line() for child in node.children: @@ -1084,8 +1084,35 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: leaf.prefix = "" +def maybe_wrap_multiple_context_managers(node: Node, mode: Mode) -> None: + if ( + Preview.wrap_multiple_context_managers_in_parens not in mode + or not supports_feature( + mode.target_versions, Feature.PARENTHESIZED_CONTEXT_MANAGERS + ) + or len(node.children) <= 2 + # If it's an atom, it's already wrapped in parens. + or node.children[1].type == syms.atom + ): + return + colon_index: Optional[int] = None + for i in range(2, len(node.children)): + if node.children[i].type == token.COLON: + colon_index = i + break + if colon_index is not None: + # TODO: test comments!! + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + children = node.children[1:colon_index] + for c in children: + c.remove() + new_child = Node(syms.atom, [lpar, Node(syms.testlist_gexp, children), rpar]) + node.insert_child(1, new_child) + + def normalize_invisible_parens( - node: Node, parens_after: Set[str], *, preview: bool + node: Node, parens_after: Set[str], *, mode: Mode ) -> None: """Make existing optional parentheses invisible or create new ones. @@ -1095,18 +1122,21 @@ def normalize_invisible_parens( Standardizes on visible parentheses for single-element tuples, and keeps existing visible parentheses for other tuples and generator expressions. """ - for pc in list_comments(node.prefix, is_endmarker=False, preview=preview): + for pc in list_comments(node.prefix, is_endmarker=False, preview=mode.preview): if pc.value in FMT_OFF: # This `node` has a prefix with `# fmt: off`, don't mess with parens. return + + # TODO: Explain. + if node.type == syms.with_stmt: + maybe_wrap_multiple_context_managers(node, mode) + check_lpar = False for index, child in enumerate(list(node.children)): # Fixes a bug where invisible parens are not properly stripped from # assignment statements that contain type annotations. if isinstance(child, Node) and child.type == syms.annassign: - normalize_invisible_parens( - child, parens_after=parens_after, preview=preview - ) + normalize_invisible_parens(child, parens_after=parens_after, mode=mode) # Add parentheses around long tuple unpacking in assignments. if ( @@ -1118,7 +1148,7 @@ def normalize_invisible_parens( if check_lpar: if ( - preview + mode.preview and child.type == syms.atom and node.type == syms.for_stmt and isinstance(child.prev_sibling, Leaf) @@ -1131,7 +1161,9 @@ def normalize_invisible_parens( remove_brackets_around_comma=True, ): wrap_in_parentheses(node, child, visible=False) - elif preview and isinstance(child, Node) and node.type == syms.with_stmt: + elif ( + mode.preview and isinstance(child, Node) and node.type == syms.with_stmt + ): remove_with_parens(child, node) elif child.type == syms.atom: if maybe_make_parens_invisible_in_atom( @@ -1142,17 +1174,7 @@ def normalize_invisible_parens( elif is_one_tuple(child): wrap_in_parentheses(node, child, visible=True) elif node.type == syms.import_from: - # "import from" nodes store parentheses directly as part of - # the statement - if is_lpar_token(child): - assert is_rpar_token(node.children[-1]) - # make parentheses invisible - child.value = "" - node.children[-1].value = "" - elif child.type != token.STAR: - # insert invisible parentheses - node.insert_child(index, Leaf(token.LPAR, "")) - node.append_child(Leaf(token.RPAR, "")) + _normalize_import_from(node, child, index) break elif ( index == 1 @@ -1167,13 +1189,27 @@ def normalize_invisible_parens( elif not (isinstance(child, Leaf) and is_multiline_string(child)): wrap_in_parentheses(node, child, visible=False) - comma_check = child.type == token.COMMA if preview else False + comma_check = child.type == token.COMMA if mode.preview else False check_lpar = isinstance(child, Leaf) and ( child.value in parens_after or comma_check ) +def _normalize_import_from(parent: Node, child: LN, index: int) -> None: + # "import from" nodes store parentheses directly as part of + # the statement + if is_lpar_token(child): + assert is_rpar_token(parent.children[-1]) + # make parentheses invisible + child.value = "" + parent.children[-1].value = "" + elif child.type != token.STAR: + # insert invisible parentheses + parent.insert_child(index, Leaf(token.LPAR, "")) + parent.append_child(Leaf(token.RPAR, "")) + + def remove_await_parens(node: Node) -> None: if node.children[0].type == token.AWAIT and len(node.children) > 1: if ( diff --git a/src/black/mode.py b/src/black/mode.py index 775805ae960..af0706e6a0b 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -50,6 +50,7 @@ class Feature(Enum): EXCEPT_STAR = 14 VARIADIC_GENERICS = 15 DEBUG_F_STRINGS = 16 + PARENTHESIZED_CONTEXT_MANAGERS = 17 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -106,6 +107,7 @@ class Feature(Enum): Feature.POS_ONLY_ARGUMENTS, Feature.UNPACKING_ON_FLOW, Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, }, TargetVersion.PY310: { Feature.F_STRINGS, @@ -120,6 +122,7 @@ class Feature(Enum): Feature.POS_ONLY_ARGUMENTS, Feature.UNPACKING_ON_FLOW, Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, Feature.PATTERN_MATCHING, }, TargetVersion.PY311: { @@ -135,6 +138,7 @@ class Feature(Enum): Feature.POS_ONLY_ARGUMENTS, Feature.UNPACKING_ON_FLOW, Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, Feature.PATTERN_MATCHING, Feature.EXCEPT_STAR, Feature.VARIADIC_GENERICS, @@ -164,6 +168,7 @@ class Preview(Enum): parenthesize_conditional_expressions = auto() skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() + wrap_multiple_context_managers_in_parens = auto() class Deprecated(UserWarning): diff --git a/tests/data/preview_38/multiple_context_managers.py b/tests/data/preview_38/multiple_context_managers.py new file mode 100644 index 00000000000..6ec4684e441 --- /dev/null +++ b/tests/data/preview_38/multiple_context_managers.py @@ -0,0 +1,38 @@ +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2(), \ + make_context_manager3() as cm3, \ + make_context_manager4() \ +: + pass + + +with \ + new_new_new1() as cm1, \ + new_new_new2() \ +: + pass + + +# output + + +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + pass + + +with make_context_manager1() as cm1, make_context_manager2(), make_context_manager3() as cm3, make_context_manager4(): + pass + + +with new_new_new1() as cm1, new_new_new2(): + pass diff --git a/tests/data/preview_39/wrap_multiple_context_managers.py b/tests/data/preview_39/wrap_multiple_context_managers.py new file mode 100644 index 00000000000..6e5aad21fc6 --- /dev/null +++ b/tests/data/preview_39/wrap_multiple_context_managers.py @@ -0,0 +1,59 @@ +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2(), \ + make_context_manager3() as cm3, \ + make_context_manager4() \ +: + pass + + +with \ + new_new_new1() as cm1, \ + new_new_new2() \ +: + pass + + +with ( + new_new_new1() as cm1, + new_new_new2() +): + pass + + +# output + + +with ( + make_context_manager1() as cm1, + make_context_manager2() as cm2, + make_context_manager3() as cm3, + make_context_manager4() as cm4, +): + pass + + +with ( + make_context_manager1() as cm1, + make_context_manager2(), + make_context_manager3() as cm3, + make_context_manager4(), +): + pass + + +with new_new_new1() as cm1, new_new_new2(): + pass + + +with new_new_new1() as cm1, new_new_new2(): + pass diff --git a/tests/test_format.py b/tests/test_format.py index 01cd61eef63..a76105c4c5d 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -44,6 +44,13 @@ def test_preview_format(filename: str) -> None: ) +@pytest.mark.parametrize("filename", all_data_cases("preview_38")) +def test_preview_targeting_python_38_format(filename: str) -> None: + source, expected = read_data("preview_38", filename) + mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY38}) + assert_format(source, expected, mode, minimum_version=(3, 8)) + + @pytest.mark.parametrize("filename", all_data_cases("preview_39")) def test_preview_minimum_python_39_format(filename: str) -> None: source, expected = read_data("preview_39", filename) From cdfb744c51c10cee7152ceaaff22a2dcc508794e Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 9 Jan 2023 21:34:12 -0800 Subject: [PATCH 2/5] Fix the version detection logic. --- src/black/__init__.py | 28 +++++++++++++- src/black/linegen.py | 27 +++++++++----- .../auto_detect/features_3_10.py | 35 ++++++++++++++++++ .../auto_detect/features_3_11.py | 37 +++++++++++++++++++ .../auto_detect/features_3_8.py | 18 +++++++++ .../auto_detect/features_3_9.py | 34 +++++++++++++++++ .../targeting_py38.py} | 0 .../targeting_py39.py} | 2 +- tests/test_format.py | 29 +++++++++++---- 9 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 tests/data/preview_context_managers/auto_detect/features_3_10.py create mode 100644 tests/data/preview_context_managers/auto_detect/features_3_11.py create mode 100644 tests/data/preview_context_managers/auto_detect/features_3_8.py create mode 100644 tests/data/preview_context_managers/auto_detect/features_3_9.py rename tests/data/{preview_38/multiple_context_managers.py => preview_context_managers/targeting_py38.py} (100%) rename tests/data/{preview_39/wrap_multiple_context_managers.py => preview_context_managers/targeting_py39.py} (97%) diff --git a/src/black/__init__.py b/src/black/__init__.py index 9f44722bfae..944ac588ca1 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1095,8 +1095,13 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) + context_manager_features = { + feature + for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + if supports_feature(versions, feature) + } normalize_fmt_off(src_node, preview=mode.preview) - lines = LineGenerator(mode=mode) + lines = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { feature @@ -1158,6 +1163,10 @@ def get_features_used( # noqa: C901 - relaxed decorator syntax; - usage of __future__ flags (annotations); - print / exec statements; + - parenthesized context managers; + - match statements; + - except* clause; + - variadic generics; """ features: Set[Feature] = set() if future_imports: @@ -1233,6 +1242,23 @@ def get_features_used( # noqa: C901 ): features.add(Feature.ANN_ASSIGN_EXTENDED_RHS) + elif ( + n.type == syms.with_stmt + and len(n.children) > 2 + and n.children[1].type == syms.atom + ): + atom_children = n.children[1].children + if ( + len(atom_children) == 3 + and atom_children[0].type == token.LPAR + and atom_children[1].type == syms.testlist_gexp + and atom_children[2].type == token.RPAR + ): + features.add(Feature.PARENTHESIZED_CONTEXT_MANAGERS) + + elif n.type == syms.match_stmt: + features.add(Feature.PATTERN_MATCHING) + elif ( n.type == syms.except_clause and len(n.children) >= 2 diff --git a/src/black/linegen.py b/src/black/linegen.py index c3e8b46a5b9..2b3856f4e5e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -22,7 +22,7 @@ is_line_short_enough, line_to_string, ) -from black.mode import Feature, Mode, Preview, supports_feature +from black.mode import Feature, Mode, Preview from black.nodes import ( ASSIGNMENTS, BRACKETS, @@ -89,8 +89,9 @@ class LineGenerator(Visitor[Line]): in ways that will no longer stringify to valid Python code on the tree. """ - def __init__(self, mode: Mode) -> None: + def __init__(self, mode: Mode, features: Collection[Feature]) -> None: self.mode = mode + self.features = features self.current_line: Line self.__post_init__() @@ -190,7 +191,9 @@ def visit_stmt( `parens` holds a set of string leaf values immediately after which invisible parens should be put. """ - normalize_invisible_parens(node, parens_after=parens, mode=self.mode) + normalize_invisible_parens( + node, parens_after=parens, mode=self.mode, features=self.features + ) for child in node.children: if is_name_token(child) and child.value in keywords: yield from self.line() @@ -243,7 +246,9 @@ def visit_funcdef(self, node: Node) -> Iterator[Line]: def visit_match_case(self, node: Node) -> Iterator[Line]: """Visit either a match or case statement.""" - normalize_invisible_parens(node, parens_after=set(), mode=self.mode) + normalize_invisible_parens( + node, parens_after=set(), mode=self.mode, features=self.features + ) yield from self.line() for child in node.children: @@ -1087,9 +1092,6 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: def maybe_wrap_multiple_context_managers(node: Node, mode: Mode) -> None: if ( Preview.wrap_multiple_context_managers_in_parens not in mode - or not supports_feature( - mode.target_versions, Feature.PARENTHESIZED_CONTEXT_MANAGERS - ) or len(node.children) <= 2 # If it's an atom, it's already wrapped in parens. or node.children[1].type == syms.atom @@ -1112,7 +1114,7 @@ def maybe_wrap_multiple_context_managers(node: Node, mode: Mode) -> None: def normalize_invisible_parens( - node: Node, parens_after: Set[str], *, mode: Mode + node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: """Make existing optional parentheses invisible or create new ones. @@ -1128,7 +1130,10 @@ def normalize_invisible_parens( return # TODO: Explain. - if node.type == syms.with_stmt: + if ( + node.type == syms.with_stmt + and Feature.PARENTHESIZED_CONTEXT_MANAGERS in features + ): maybe_wrap_multiple_context_managers(node, mode) check_lpar = False @@ -1136,7 +1141,9 @@ def normalize_invisible_parens( # Fixes a bug where invisible parens are not properly stripped from # assignment statements that contain type annotations. if isinstance(child, Node) and child.type == syms.annassign: - normalize_invisible_parens(child, parens_after=parens_after, mode=mode) + normalize_invisible_parens( + child, parens_after=parens_after, mode=mode, features=features + ) # Add parentheses around long tuple unpacking in assignments. if ( diff --git a/tests/data/preview_context_managers/auto_detect/features_3_10.py b/tests/data/preview_context_managers/auto_detect/features_3_10.py new file mode 100644 index 00000000000..1458df1cb41 --- /dev/null +++ b/tests/data/preview_context_managers/auto_detect/features_3_10.py @@ -0,0 +1,35 @@ +# This file uses pattern matching introduced in Python 3.10. + + +match http_code: + case 404: + print("Not found") + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +# output + + +# This file uses pattern matching introduced in Python 3.10. + + +match http_code: + case 404: + print("Not found") + + +with ( + make_context_manager1() as cm1, + make_context_manager2() as cm2, + make_context_manager3() as cm3, + make_context_manager4() as cm4, +): + pass diff --git a/tests/data/preview_context_managers/auto_detect/features_3_11.py b/tests/data/preview_context_managers/auto_detect/features_3_11.py new file mode 100644 index 00000000000..f83c5330ab3 --- /dev/null +++ b/tests/data/preview_context_managers/auto_detect/features_3_11.py @@ -0,0 +1,37 @@ +# This file uses except* clause in Python 3.11. + + +try: + some_call() +except* Error as e: + pass + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +# output + + +# This file uses except* clause in Python 3.11. + + +try: + some_call() +except* Error as e: + pass + + +with ( + make_context_manager1() as cm1, + make_context_manager2() as cm2, + make_context_manager3() as cm3, + make_context_manager4() as cm4, +): + pass diff --git a/tests/data/preview_context_managers/auto_detect/features_3_8.py b/tests/data/preview_context_managers/auto_detect/features_3_8.py new file mode 100644 index 00000000000..79cc9ae8af8 --- /dev/null +++ b/tests/data/preview_context_managers/auto_detect/features_3_8.py @@ -0,0 +1,18 @@ +# This file doesn't used any Python 3.9+ only grammars. + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +# output +# This file doesn't used any Python 3.9+ only grammars. + + +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + pass diff --git a/tests/data/preview_context_managers/auto_detect/features_3_9.py b/tests/data/preview_context_managers/auto_detect/features_3_9.py new file mode 100644 index 00000000000..0d28f993108 --- /dev/null +++ b/tests/data/preview_context_managers/auto_detect/features_3_9.py @@ -0,0 +1,34 @@ +# This file uses parenthesized context managers introduced in Python 3.9. + + +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + pass + + +with ( + new_new_new1() as cm1, + new_new_new2() +): + pass + + +# output +# This file uses parenthesized context managers introduced in Python 3.9. + + +with ( + make_context_manager1() as cm1, + make_context_manager2() as cm2, + make_context_manager3() as cm3, + make_context_manager4() as cm4, +): + pass + + +with new_new_new1() as cm1, new_new_new2(): + pass diff --git a/tests/data/preview_38/multiple_context_managers.py b/tests/data/preview_context_managers/targeting_py38.py similarity index 100% rename from tests/data/preview_38/multiple_context_managers.py rename to tests/data/preview_context_managers/targeting_py38.py diff --git a/tests/data/preview_39/wrap_multiple_context_managers.py b/tests/data/preview_context_managers/targeting_py39.py similarity index 97% rename from tests/data/preview_39/wrap_multiple_context_managers.py rename to tests/data/preview_context_managers/targeting_py39.py index 6e5aad21fc6..18ee1e4a444 100644 --- a/tests/data/preview_39/wrap_multiple_context_managers.py +++ b/tests/data/preview_context_managers/targeting_py39.py @@ -25,7 +25,7 @@ with ( new_new_new1() as cm1, - new_new_new2() + new_new_new2() ): pass diff --git a/tests/test_format.py b/tests/test_format.py index a76105c4c5d..4a6ad5ac965 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,4 +1,5 @@ from dataclasses import replace +import re from typing import Any, Iterator from unittest.mock import patch @@ -44,13 +45,6 @@ def test_preview_format(filename: str) -> None: ) -@pytest.mark.parametrize("filename", all_data_cases("preview_38")) -def test_preview_targeting_python_38_format(filename: str) -> None: - source, expected = read_data("preview_38", filename) - mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY38}) - assert_format(source, expected, mode, minimum_version=(3, 8)) - - @pytest.mark.parametrize("filename", all_data_cases("preview_39")) def test_preview_minimum_python_39_format(filename: str) -> None: source, expected = read_data("preview_39", filename) @@ -65,6 +59,27 @@ def test_preview_minimum_python_310_format(filename: str) -> None: assert_format(source, expected, mode, minimum_version=(3, 10)) +def test_preview_context_managers_targeting_py38() -> None: + source, expected = read_data("preview_context_managers", "targeting_py38.py") + mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY38}) + assert_format(source, expected, mode, minimum_version=(3, 8)) + + +def test_preview_context_managers_targeting_py39() -> None: + source, expected = read_data("preview_context_managers", "targeting_py39.py") + mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY39}) + assert_format(source, expected, mode, minimum_version=(3, 9)) + + +@pytest.mark.parametrize("filename", all_data_cases("preview_context_managers/auto_detect")) +def test_preview_context_managers_auto_detect(filename: str) -> None: + match = re.match(r"features_3_(\d+)", filename) + assert match is not None, "Unexpected filename format: %s" % filename + source, expected = read_data("preview_context_managers/auto_detect", filename) + mode = black.Mode(preview=True) + assert_format(source, expected, mode, minimum_version=(3, int(match.group(1)))) + + # =============== # # Complex cases # ============= # From a378a9f4666ac8410072d7ad422a45a98c163dec Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 9 Jan 2023 21:53:08 -0800 Subject: [PATCH 3/5] Add docs/comments, and more test cases. --- src/black/linegen.py | 78 ++++++++++++------- .../targeting_py39.py | 45 +++++++++++ tests/test_format.py | 6 +- 3 files changed, 97 insertions(+), 32 deletions(-) diff --git a/src/black/linegen.py b/src/black/linegen.py index 2b3856f4e5e..5e9e403448b 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1089,30 +1089,6 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: leaf.prefix = "" -def maybe_wrap_multiple_context_managers(node: Node, mode: Mode) -> None: - if ( - Preview.wrap_multiple_context_managers_in_parens not in mode - or len(node.children) <= 2 - # If it's an atom, it's already wrapped in parens. - or node.children[1].type == syms.atom - ): - return - colon_index: Optional[int] = None - for i in range(2, len(node.children)): - if node.children[i].type == token.COLON: - colon_index = i - break - if colon_index is not None: - # TODO: test comments!! - lpar = Leaf(token.LPAR, "") - rpar = Leaf(token.RPAR, "") - children = node.children[1:colon_index] - for c in children: - c.remove() - new_child = Node(syms.atom, [lpar, Node(syms.testlist_gexp, children), rpar]) - node.insert_child(1, new_child) - - def normalize_invisible_parens( node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: @@ -1129,12 +1105,11 @@ def normalize_invisible_parens( # This `node` has a prefix with `# fmt: off`, don't mess with parens. return - # TODO: Explain. - if ( - node.type == syms.with_stmt - and Feature.PARENTHESIZED_CONTEXT_MANAGERS in features - ): - maybe_wrap_multiple_context_managers(node, mode) + # The multiple context managers grammar has a different pattern, thus this is + # separate from the for-loop below. This possibly wraps them in invisible parens, + # and later will be removed in remove_with_parens when needed. + if node.type == syms.with_stmt: + _maybe_wrap_cms_in_parens(node, mode, features) check_lpar = False for index, child in enumerate(list(node.children)): @@ -1253,6 +1228,49 @@ def remove_await_parens(node: Node) -> None: remove_await_parens(bracket_contents) +def _maybe_wrap_cms_in_parens( + node: Node, mode: Mode, features: Collection[Feature] +) -> None: + """When enabled and safe, wrap the multiple context managers in invisible parens. + + It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS. + """ + if ( + Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features + or Preview.wrap_multiple_context_managers_in_parens not in mode + or len(node.children) <= 2 + # If it's an atom, it's already wrapped in parens. + or node.children[1].type == syms.atom + ): + return + colon_index: Optional[int] = None + for i in range(2, len(node.children)): + if node.children[i].type == token.COLON: + colon_index = i + break + if colon_index is not None: + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + context_managers = node.children[1:colon_index] + for child in context_managers: + child.remove() + # After wrapping, the with_stmt will look like this: + # with_stmt + # NAME 'with' + # atom + # LPAR '' + # testlist_gexp + # ... <-- context_managers + # /testlist_gexp + # RPAR '' + # /atom + # COLON ':' + new_child = Node( + syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar] + ) + node.insert_child(1, new_child) + + def remove_with_parens(node: Node, parent: Node) -> None: """Recursively hide optional parens in `with` statements.""" # Removing all unnecessary parentheses in with statements in one pass is a tad diff --git a/tests/data/preview_context_managers/targeting_py39.py b/tests/data/preview_context_managers/targeting_py39.py index 18ee1e4a444..5cb8763040a 100644 --- a/tests/data/preview_context_managers/targeting_py39.py +++ b/tests/data/preview_context_managers/targeting_py39.py @@ -7,6 +7,7 @@ pass +# Leading comment with \ make_context_manager1() as cm1, \ make_context_manager2(), \ @@ -30,6 +31,24 @@ pass +# Leading comment. +with ( + # First comment. + new_new_new1() as cm1, + # Second comment. + new_new_new2() + # Last comment. +): + pass + + +with \ + this_is_a_very_long_call(looong_arg1=looong_value1, looong_arg2=looong_value2) as cm1, \ + this_is_a_very_long_call(looong_arg1=looong_value1, looong_arg2=looong_value2, looong_arg3=looong_value3, looong_arg4=looong_value4) as cm2 \ +: + pass + + # output @@ -42,6 +61,7 @@ pass +# Leading comment with ( make_context_manager1() as cm1, make_context_manager2(), @@ -57,3 +77,28 @@ with new_new_new1() as cm1, new_new_new2(): pass + + +# Leading comment. +with ( + # First comment. + new_new_new1() as cm1, + # Second comment. + new_new_new2() + # Last comment. +): + pass + + +with ( + this_is_a_very_long_call( + looong_arg1=looong_value1, looong_arg2=looong_value2 + ) as cm1, + this_is_a_very_long_call( + looong_arg1=looong_value1, + looong_arg2=looong_value2, + looong_arg3=looong_value3, + looong_arg4=looong_value4, + ) as cm2, +): + pass diff --git a/tests/test_format.py b/tests/test_format.py index 4a6ad5ac965..ac3465a0ed3 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,5 +1,5 @@ -from dataclasses import replace import re +from dataclasses import replace from typing import Any, Iterator from unittest.mock import patch @@ -71,7 +71,9 @@ def test_preview_context_managers_targeting_py39() -> None: assert_format(source, expected, mode, minimum_version=(3, 9)) -@pytest.mark.parametrize("filename", all_data_cases("preview_context_managers/auto_detect")) +@pytest.mark.parametrize( + "filename", all_data_cases("preview_context_managers/auto_detect") +) def test_preview_context_managers_auto_detect(filename: str) -> None: match = re.match(r"features_3_(\d+)", filename) assert match is not None, "Unexpected filename format: %s" % filename From b1ca32276b74eb97702c5ebc41a28186cdd54c1a Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 9 Jan 2023 22:09:34 -0800 Subject: [PATCH 4/5] Update CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 2da0fb4720c..6bdabcda85a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,7 @@ string lambda values are now wrapped in parentheses (#3440) - Exclude string type annotations from improved string processing; fix crash when the return type annotation is stringified and spans across multiple lines (#3462) +- Wrap multiple context managers in parentheses when targeting Python 3.9+ (#3489) ### Configuration From a81d7b4692e661b277648b2725e52e851e4729c8 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Tue, 10 Jan 2023 16:07:04 -0800 Subject: [PATCH 5/5] Test that something like with (a): pass doesn't get autodetected as 3.9+ --- .../auto_detect/features_3_8.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/data/preview_context_managers/auto_detect/features_3_8.py b/tests/data/preview_context_managers/auto_detect/features_3_8.py index 79cc9ae8af8..e05094e1421 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_8.py +++ b/tests/data/preview_context_managers/auto_detect/features_3_8.py @@ -1,4 +1,10 @@ -# This file doesn't used any Python 3.9+ only grammars. +# This file doesn't use any Python 3.9+ only grammars. + + +# Make sure parens around a single context manager don't get autodetected as +# Python 3.9+. +with (a): + pass with \ @@ -11,7 +17,13 @@ # output -# This file doesn't used any Python 3.9+ only grammars. +# This file doesn't use any Python 3.9+ only grammars. + + +# Make sure parens around a single context manager don't get autodetected as +# Python 3.9+. +with a: + pass with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: