From f2a54eea7970a748f9ba11dcf6488c1061704858 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sat, 6 Nov 2021 14:19:05 -0400 Subject: [PATCH 1/2] Improve Python 2 only syntax detection First of all this fixes a mistake I made in Python 2 deprecation PR using token.* to check for print/exec statements. Turns out that for nodes with a type value higher than 256 its numeric type isn't guaranteed to be constant. Using syms.* instead fixes this. Also add support for the following cases: print "hello, world!" exec "print('hello, world!')" def set_position((x, y), value): pass try: pass except Exception, err: pass raise RuntimeError, "I feel like crashing today :p" `wow_these_really_did_exist` 10L --- CHANGES.md | 7 ++++ src/black/__init__.py | 31 ++++++++++++++-- src/black/mode.py | 10 +++++ src/blib2to3/pgen2/token.py | 3 -- tests/data/python2_detection.py | 66 +++++++++++++++++++++++++++++++++ tests/test_black.py | 13 +++++++ 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 tests/data/python2_detection.py diff --git a/CHANGES.md b/CHANGES.md index b454a73edab..215680ff2ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # Change Log +## _Unreleased_ + +### _Black_ + +- Warn about Python 2 deprecation in more cases by improving Python 2 only syntax + detection (#2592) + ## 21.10b0 ### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index ba4d3dea70e..06295c412e4 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1132,8 +1132,12 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 features.add(Feature.F_STRINGS) elif n.type == token.NUMBER: - if "_" in n.value: # type: ignore + assert isinstance(n, Leaf) + if "_" in n.value: features.add(Feature.NUMERIC_UNDERSCORES) + elif n.value.endswith("L"): + # Python 2: 10L + features.add(Feature.LONG_INT_LITERAL) elif n.type == token.SLASH: if n.parent and n.parent.type in { @@ -1171,10 +1175,31 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 if argch.type in STARS: features.add(feature) - elif n.type == token.PRINT_STMT: + # Python 2 only features (for its deprecation) except for 10L, see above + elif n.type == syms.print_stmt: features.add(Feature.PRINT_STMT) - elif n.type == token.EXEC_STMT: + elif n.type == syms.exec_stmt: features.add(Feature.EXEC_STMT) + elif n.type == syms.tfpdef: + # def set_position((x, y), value): + # ... + features.add(Feature.AUTOMATIC_PARAMETER_UNPACKING) + elif n.type == syms.except_clause: + # try: + # ... + # except Exception, err: + # ... + if len(n.children) >= 4: + if n.children[-2].type == token.COMMA: + features.add(Feature.COMMA_STYLE_EXCEPT) + elif n.type == syms.raise_stmt: + # raise Exception, "msg" + if len(n.children) >= 4: + if n.children[-2].type == token.COMMA: + features.add(Feature.COMMA_STYLE_RAISE) + elif n.type == token.BACKQUOTE: + # `i'm surprised this ever existed` + features.add(Feature.BACKQUOTE_REPR) return features diff --git a/src/black/mode.py b/src/black/mode.py index 374c47a42eb..5fd179f4726 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -44,6 +44,11 @@ class Feature(Enum): # temporary for Python 2 deprecation PRINT_STMT = 200 EXEC_STMT = 201 + AUTOMATIC_PARAMETER_UNPACKING = 202 + COMMA_STYLE_EXCEPT = 203 + COMMA_STYLE_RAISE = 204 + LONG_INT_LITERAL = 205 + BACKQUOTE_REPR = 206 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { @@ -51,6 +56,11 @@ class Feature(Enum): Feature.ASYNC_IDENTIFIERS, Feature.PRINT_STMT, Feature.EXEC_STMT, + Feature.AUTOMATIC_PARAMETER_UNPACKING, + Feature.COMMA_STYLE_EXCEPT, + Feature.COMMA_STYLE_RAISE, + Feature.LONG_INT_LITERAL, + Feature.BACKQUOTE_REPR, }, TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 349ba8023a2..1e0dec9c714 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -74,9 +74,6 @@ COLONEQUAL: Final = 59 N_TOKENS: Final = 60 NT_OFFSET: Final = 256 -# temporary for Python 2 deprecation -PRINT_STMT: Final = 316 -EXEC_STMT: Final = 288 # --end constants-- tok_name: Final[Dict[int, str]] = {} diff --git a/tests/data/python2_detection.py b/tests/data/python2_detection.py new file mode 100644 index 00000000000..b75f62fe10a --- /dev/null +++ b/tests/data/python2_detection.py @@ -0,0 +1,66 @@ +# This uses a similar construction to the decorators.py test data file FYI. + +print "hello, world!" + +### + +exec "print('hello, world!')" + +### + +def set_position((x, y), value): + pass + +### + +try: + pass +except Exception, err: + pass + +### + +raise RuntimeError, "I feel like crashing today :p" + +### + +`wow_these_really_did_exist` + +### + +10L + +# output + +print("hello python three!") + +### + +exec("I'm not sure if you can use exec like this but that's not important here!") + +### + +try: + pass +except make_exception(1, 2): + pass + +### + +try: + pass +except Exception as err: + pass + +### + +raise RuntimeError(make_msg(1, 2)) + +### + +def set_position(x, y, value): + pass + +### + +10 diff --git a/tests/test_black.py b/tests/test_black.py index b96a5438557..7f8fff3cabc 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2017,6 +2017,7 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: ) +@pytest.mark.python2 @pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"]) def test_python_2_deprecation_with_target_version(explicit: bool) -> None: args = [ @@ -2032,6 +2033,18 @@ def test_python_2_deprecation_with_target_version(explicit: bool) -> None: assert "DEPRECATION: Python 2 support will be removed" in result.stderr +@pytest.mark.python2 +def test_python_2_deprecation_autodetection_extended() -> None: + # this test has a similar construction to test_get_features_used_decorator + python2, non_python2 = read_data("python2_detection") + for python2_case in python2.split("###"): + node = black.lib2to3_parse(python2_case) + assert black.detect_target_versions(node) == {TargetVersion.PY27} + for non_python2_case in non_python2.split("###"): + node = black.lib2to3_parse(non_python2_case) + assert black.detect_target_versions(node) != {TargetVersion.PY27} + + with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() From 78d32bf5f4c1d3fe0db9e61badba7895facb096b Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 8 Nov 2021 18:29:36 -0500 Subject: [PATCH 2/2] Add octal support, more test cases, and fixup long ints Co-authored-by: Jelle Zijlstra --- src/black/__init__.py | 9 +++++++-- src/black/mode.py | 4 +++- tests/data/python2_detection.py | 24 ++++++++++++++++++++++++ tests/test_black.py | 6 ++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 06295c412e4..ad4ee1a0d1a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1135,9 +1135,14 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 assert isinstance(n, Leaf) if "_" in n.value: features.add(Feature.NUMERIC_UNDERSCORES) - elif n.value.endswith("L"): + elif n.value.endswith(("L", "l")): # Python 2: 10L features.add(Feature.LONG_INT_LITERAL) + elif len(n.value) >= 2 and n.value[0] == "0" and n.value[1].isdigit(): + # Python 2: 0123; 00123; ... + if not all(char == "0" for char in n.value): + # although we don't want to match 0000 or similar + features.add(Feature.OCTAL_INT_LITERAL) elif n.type == token.SLASH: if n.parent and n.parent.type in { @@ -1175,7 +1180,7 @@ def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 if argch.type in STARS: features.add(feature) - # Python 2 only features (for its deprecation) except for 10L, see above + # Python 2 only features (for its deprecation) except for integers, see above elif n.type == syms.print_stmt: features.add(Feature.PRINT_STMT) elif n.type == syms.exec_stmt: diff --git a/src/black/mode.py b/src/black/mode.py index 5fd179f4726..01ee336366c 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -48,7 +48,8 @@ class Feature(Enum): COMMA_STYLE_EXCEPT = 203 COMMA_STYLE_RAISE = 204 LONG_INT_LITERAL = 205 - BACKQUOTE_REPR = 206 + OCTAL_INT_LITERAL = 206 + BACKQUOTE_REPR = 207 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { @@ -60,6 +61,7 @@ class Feature(Enum): Feature.COMMA_STYLE_EXCEPT, Feature.COMMA_STYLE_RAISE, Feature.LONG_INT_LITERAL, + Feature.OCTAL_INT_LITERAL, Feature.BACKQUOTE_REPR, }, TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, diff --git a/tests/data/python2_detection.py b/tests/data/python2_detection.py index b75f62fe10a..8de2bb58adc 100644 --- a/tests/data/python2_detection.py +++ b/tests/data/python2_detection.py @@ -30,6 +30,14 @@ def set_position((x, y), value): 10L +### + +10l + +### + +0123 + # output print("hello python three!") @@ -58,9 +66,25 @@ def set_position((x, y), value): ### +raise RuntimeError("boom!",) + +### + def set_position(x, y, value): pass ### 10 + +### + +0 + +### + +000 + +### + +0o12 \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index 7f8fff3cabc..7dbc3809d26 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2039,10 +2039,12 @@ def test_python_2_deprecation_autodetection_extended() -> None: python2, non_python2 = read_data("python2_detection") for python2_case in python2.split("###"): node = black.lib2to3_parse(python2_case) - assert black.detect_target_versions(node) == {TargetVersion.PY27} + assert black.detect_target_versions(node) == {TargetVersion.PY27}, python2_case for non_python2_case in non_python2.split("###"): node = black.lib2to3_parse(non_python2_case) - assert black.detect_target_versions(node) != {TargetVersion.PY27} + assert black.detect_target_versions(node) != { + TargetVersion.PY27 + }, non_python2_case with open(black.__file__, "r", encoding="utf-8") as _bf: