From 2726ce822ca55c6569f2c21aa8d38de6e33b1c3f Mon Sep 17 00:00:00 2001 From: olliemath Date: Thu, 21 Oct 2021 17:03:56 +0100 Subject: [PATCH 01/20] Allow install under pypy --- setup.py | 2 +- src/black/parsing.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index de84dc37bb8..a0c2006ef33 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_long_description() -> str: "click>=7.1.2", "platformdirs>=2", "tomli>=0.2.6,<2.0.0", - "typed-ast>=1.4.2; python_version < '3.8'", + "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", "regex>=2020.1.8", "pathspec>=0.9.0, <1", "dataclasses>=0.6; python_version < '3.7'", diff --git a/src/black/parsing.py b/src/black/parsing.py index 0b8d984cedd..7018b628043 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -15,10 +15,13 @@ from black.mode import TargetVersion, Feature, supports_feature from black.nodes import syms +_IS_PYPY = "__pypy__" in sys.modules + try: from typed_ast import ast3, ast27 except ImportError: - if sys.version_info < (3, 8): + # Either our python version is too low, or we're on pypy + if sys.version_info < (3, 8) and not _IS_PYPY: print( "The typed_ast package is required but not installed.\n" "You can upgrade to Python 3.8+ or install typed_ast with\n" @@ -114,7 +117,10 @@ def parse_single_version( if sys.version_info >= (3, 8) and version >= (3,): return ast.parse(src, filename, feature_version=version) elif version >= (3,): - return ast3.parse(src, filename, feature_version=version[1]) + if _IS_PYPY: + return ast3.parse(src, filename) + else: + return ast3.parse(src, filename, feature_version=version[1]) elif version == (2, 7): return ast27.parse(src) raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!") @@ -148,12 +154,14 @@ def stringify_ast( yield f"{' ' * depth}{node.__class__.__name__}(" for field in sorted(node._fields): # noqa: F402 - # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): - break + # TypeIgnore will not be present using pypy < 3.8, so need for this + if not (_IS_PYPY and sys.version_info < (3, 8)): + # TypeIgnore has only one field 'lineno' which breaks this comparison + type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore) + if sys.version_info >= (3, 8): + type_ignore_classes += (ast.TypeIgnore,) + if isinstance(node, type_ignore_classes): + break try: value = getattr(node, field) From 6ae0833ce604c810c628915c00c43bb597e277e2 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 22 Oct 2021 11:25:20 +0000 Subject: [PATCH 02/20] Add pypy3 test action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 296ac34a3fb..05e65658426 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pyyp-3.6", "pypy-3.7"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: From 648f1c99070271e278966297062a103f28d8fa40 Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 22 Oct 2021 12:25:59 +0100 Subject: [PATCH 03/20] Compat: handle bad cache items under pypy PyPy raises an IndexError unpickling bad pickle files. --- src/black/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/cache.py b/src/black/cache.py index 3f165de2ed6..bca7279f990 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -35,7 +35,7 @@ def read_cache(mode: Mode) -> Cache: with cache_file.open("rb") as fobj: try: cache: Cache = pickle.load(fobj) - except (pickle.UnpicklingError, ValueError): + except (pickle.UnpicklingError, ValueError, IndexError): return {} return cache From 756a12954aff289360492000d90045d0690fbce7 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 22 Oct 2021 12:24:50 +0000 Subject: [PATCH 04/20] Update changes for #2559 --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 864f0a54410..06a7ca74339 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - Add new `--workers` parameter (#2514) - Fixed feature detection for positional-only arguments in lambdas (#2532) - Bumped typed-ast version minimum to 1.4.3 for 3.10 compatiblity (#2519) +- Add experimental pypy support (#2559) ### _Blackd_ From 88cada42fd81b89b0e2a4c6c284e3f381b87de2c Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 22 Oct 2021 14:27:54 +0100 Subject: [PATCH 05/20] Fix typo in .github/workflows/test.yml Co-authored-by: Jelle Zijlstra --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05e65658426..3c471dd8128 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pyyp-3.6", "pypy-3.7"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.6", "pypy-3.7"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: From db928b8f570bd6dadd77308e68d5f2abab26f83f Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 22 Oct 2021 14:51:44 +0100 Subject: [PATCH 06/20] Support only pypy3.7+ Both unit-tests and mypy fail for pypy3.6. As such, it's likely that far too many changes would be needed to justify supporting the older version. --- src/black/parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 7018b628043..3710066e9ea 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -21,7 +21,7 @@ from typed_ast import ast3, ast27 except ImportError: # Either our python version is too low, or we're on pypy - if sys.version_info < (3, 8) and not _IS_PYPY: + if sys.version_info < (3, 7) or (sys.version_info < (3, 8) and not _IS_PYPY): print( "The typed_ast package is required but not installed.\n" "You can upgrade to Python 3.8+ or install typed_ast with\n" From 2279abfb6dabf8e5a9acea0b17bbc803fb5cb465 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 22 Oct 2021 14:13:40 +0000 Subject: [PATCH 07/20] Testing: only support pypy3.7+ The older ast does not support quite a few unicode edge cases and is disliked by mypy. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c471dd8128..d30c162799b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.6", "pypy-3.7"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: From 8efb3aab2fe27b1a23900a962ab8f3f0a666f3bb Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 22 Oct 2021 18:03:16 +0100 Subject: [PATCH 08/20] Tox: create a pypy3 section --- tox.ini | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 57f41acb3d1..683a5439ea9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {,ci-}py{36,37,38,39,310},fuzz +envlist = {,ci-}py{36,37,38,39,310,py3},fuzz [testenv] setenv = PYTHONPATH = {toxinidir}/src @@ -31,6 +31,31 @@ commands = --cov --cov-append {posargs} coverage report +[testenv:{,ci-}pypy3] +setenv = PYTHONPATH = {toxinidir}/src +skip_install = True +recreate = True +deps = + -r{toxinidir}/test_requirements.txt +; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317 +; this seems to cause tox to wait forever +; remove this when pypy releases the bugfix +commands = + pip install -e .[d] + coverage erase + pytest tests --run-optional no_python2 \ + --run-optional no_jupyter \ + !ci: --numprocesses auto \ + ci: --numprocesses 1 \ + --cov {posargs} + pip install -e .[jupyter] + pytest tests --run-optional jupyter \ + -m jupyter \ + !ci: --numprocesses auto \ + ci: --numprocesses 1 \ + --cov --cov-append {posargs} + coverage report + [testenv:fuzz] skip_install = True deps = From 5885d6342f4abcbaa5699889c9dcea16783bd9ea Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 22 Oct 2021 17:10:21 +0000 Subject: [PATCH 09/20] Update github actions for pypy We have to pass information about the interpreter to tox, so that we can skip the python2 tests --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d30c162799b..38ae4a72555 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,9 +41,15 @@ jobs: python -m pip install --upgrade tox - name: Unit tests + if: "!startsWith(matrix.python-version, 'pypy')" run: | tox -e ci-py -- -v --color=yes + - name: Unit tests pypy + if: "startsWith(matrix.python-version, 'pypy')" + run: | + tox -e ci-pypy3 -- -v --color=yes + - name: Publish coverage to Coveralls # If pushed / is a pull request against main repo AND # we're running on Linux (this action only supports Linux) From fe1d91d355df5ef8718407ffdef91591ba70e39a Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Wed, 27 Oct 2021 10:10:28 +0000 Subject: [PATCH 10/20] Use python_implementation to detect pypy --- src/black/parsing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index 3710066e9ea..2188e3e56c4 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -2,6 +2,7 @@ Parse Python code and perform AST validation. """ import ast +import platform import sys from typing import Iterable, Iterator, List, Set, Union, Tuple @@ -15,7 +16,7 @@ from black.mode import TargetVersion, Feature, supports_feature from black.nodes import syms -_IS_PYPY = "__pypy__" in sys.modules +_IS_PYPY = platform.python_implementation() == "PyPy" try: from typed_ast import ast3, ast27 From 6c66ab3e58707e749b5a4984d19bbaf33a2dba53 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Wed, 27 Oct 2021 10:18:22 +0000 Subject: [PATCH 11/20] Tox: simplify pypy command Previously in CI, pypy tests would hang forever due to https://foss.heptapod.net/pypy/pypy/-/issues/3317. This was fixed in the latest release, so we can remove the workaround. --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index 683a5439ea9..465e499c907 100644 --- a/tox.ini +++ b/tox.ini @@ -37,22 +37,17 @@ skip_install = True recreate = True deps = -r{toxinidir}/test_requirements.txt -; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317 -; this seems to cause tox to wait forever -; remove this when pypy releases the bugfix commands = pip install -e .[d] coverage erase pytest tests --run-optional no_python2 \ --run-optional no_jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ --cov {posargs} pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ --cov --cov-append {posargs} coverage report From 39bfb2883f6aa50a2c7050beecada3d5c9809e3b Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Wed, 27 Oct 2021 10:34:11 +0000 Subject: [PATCH 12/20] Add PyPy section to FAQ --- docs/faq.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index c361addf7ae..8ec657556fd 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -82,3 +82,9 @@ influence their behavior. While Black does its best to recognize such comments a them in the right place, this detection is not and cannot be perfect. Therefore, you'll sometimes have to manually move these comments to the right place after you format your codebase with _Black_. + +## Can I run black with PyPy? + +Yes, there is _experimental_ support for PyPy 3.7 and higher. You cannot format Python 2 +files under PyPy, because PyPy's inbuilt ast module does not support this. For the most +stable user-experience, we recommend using CPython if possible. From c35ad0304535aae0210a1880a5bb22f019a62d6d Mon Sep 17 00:00:00 2001 From: olliemath Date: Thu, 28 Oct 2021 13:52:13 +0100 Subject: [PATCH 13/20] Revert "Tox: simplify pypy command" This reverts commit 6c66ab3e58707e749b5a4984d19bbaf33a2dba53. Even though this bug is not present for newer versions of pypy, the github action tends to receive older versions (7.3.5 on macos and 7.3.6 on windows/linux) --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 465e499c907..683a5439ea9 100644 --- a/tox.ini +++ b/tox.ini @@ -37,17 +37,22 @@ skip_install = True recreate = True deps = -r{toxinidir}/test_requirements.txt +; a separate worker is required in ci due to https://foss.heptapod.net/pypy/pypy/-/issues/3317 +; this seems to cause tox to wait forever +; remove this when pypy releases the bugfix commands = pip install -e .[d] coverage erase pytest tests --run-optional no_python2 \ --run-optional no_jupyter \ !ci: --numprocesses auto \ + ci: --numprocesses 1 \ --cov {posargs} pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ !ci: --numprocesses auto \ + ci: --numprocesses 1 \ --cov --cov-append {posargs} coverage report From 8e75e5a2ce02e14be021c63f1cfb5d440cd0454d Mon Sep 17 00:00:00 2001 From: olliemath Date: Thu, 28 Oct 2021 13:59:25 +0100 Subject: [PATCH 14/20] Testing: handle windows symlinks under pypy --- tests/test_black.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_black.py b/tests/test_black.py index 1fc63c942e9..2544f4fd0b7 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -943,7 +943,7 @@ def test_broken_symlink(self) -> None: symlink = workspace / "broken_link.py" try: symlink.symlink_to("nonexistent.py") - except OSError as e: + except (OSError, NotImplementedError) as e: self.skipTest(f"Can't create symlinks: {e}") self.invokeBlack([str(workspace.resolve())]) From da70ee630b760f4616903c42490de76baf663cb6 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Thu, 28 Oct 2021 13:27:27 +0000 Subject: [PATCH 15/20] Prettier test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38ae4a72555..7ba2a84d049 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,7 +49,7 @@ jobs: if: "startsWith(matrix.python-version, 'pypy')" run: | tox -e ci-pypy3 -- -v --color=yes - + - name: Publish coverage to Coveralls # If pushed / is a pull request against main repo AND # we're running on Linux (this action only supports Linux) From 7a2488555673ff035589b815ac8b2070e396378a Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 29 Oct 2021 15:38:00 +0000 Subject: [PATCH 16/20] Fix bad merge of changes --- CHANGES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 25ddb0b3fd5..50acc45283a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,6 @@ - Add new `--workers` parameter (#2514) - Fixed feature detection for positional-only arguments in lambdas (#2532) - Bumped typed-ast version minimum to 1.4.3 for 3.10 compatiblity (#2519) -- Add primer support for --projects (#2555) - Add experimental pypy support (#2559) ### _Blackd_ From 1677ddbf09e90ac8cdf8cc19cf35138bf6c2c133 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Sun, 31 Oct 2021 10:15:11 +0000 Subject: [PATCH 17/20] Tighten up PyPy faq --- docs/faq.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index c622f904530..4f61c09eb08 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -91,6 +91,5 @@ codebase with _Black_. ## Can I run black with PyPy? -Yes, there is _experimental_ support for PyPy 3.7 and higher. You cannot format Python 2 -files under PyPy, because PyPy's inbuilt ast module does not support this. For the most -stable user-experience, we recommend using CPython if possible. +Yes, there is support for PyPy 3.7 and higher. You cannot format Python 2 +files under PyPy, because PyPy's inbuilt ast module does not support this. From 640a1923524b72505691f1af8320f618eced19ae Mon Sep 17 00:00:00 2001 From: olliemath Date: Sun, 31 Oct 2021 23:34:00 +0000 Subject: [PATCH 18/20] Prettier FAQ --- docs/faq.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 4f61c09eb08..f0a17e34210 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -91,5 +91,5 @@ codebase with _Black_. ## Can I run black with PyPy? -Yes, there is support for PyPy 3.7 and higher. You cannot format Python 2 -files under PyPy, because PyPy's inbuilt ast module does not support this. +Yes, there is support for PyPy 3.7 and higher. You cannot format Python 2 files under +PyPy, because PyPy's inbuilt ast module does not support this. From 3215f3c7605cb8e0dc9690ac4b4590463a923fc1 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 12 Nov 2021 12:46:51 -0500 Subject: [PATCH 19/20] Move changelog to unreleased section --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8920fa468be..d954aaa13ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - Warn about Python 2 deprecation in more cases by improving Python 2 only syntax detection (#2592) +- Add experimental pypy support (#2559) ## 21.10b0 @@ -18,7 +19,6 @@ - Fixed a Python 3.10 compatibility issue where the loop argument was still being passed even though it has been removed (#2580) - Deprecate Python 2 formatting support (#2523) -- Add experimental pypy support (#2559) ### _Blackd_ From f345ef63b511d95fac37b92471182a994fdfcf45 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 13 Nov 2021 19:45:32 -0800 Subject: [PATCH 20/20] Update CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3ac603a4095..c8b9c849815 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ - Warn about Python 2 deprecation in more cases by improving Python 2 only syntax detection (#2592) -- Add experimental pypy support (#2559) +- Add experimental PyPy support (#2559) - Add partial support for the match statement. As it's experimental, it's only enabled when `--target-version py310` is explicitly specified (#2586) - Add support for parenthesized with (#2586)