From 9cbf1f162261b64ebb150639b608be0c38f23e2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Apr 2021 11:07:57 -0700 Subject: [PATCH 01/81] Bump urllib3 from 1.26.3 to 1.26.4 (#2090) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.3 to 1.26.4. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.3...1.26.4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Pipfile.lock | 89 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 9cdcba2a0ea..3d908dc90f3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -569,20 +569,20 @@ }, "cryptography": { "hashes": [ - "sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b", - "sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336", - "sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87", - "sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7", - "sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799", - "sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b", - "sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df", - "sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0", - "sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3", - "sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724", - "sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2", - "sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964" - ], - "version": "==3.4.6" + "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", + "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", + "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", + "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", + "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", + "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", + "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", + "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", + "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", + "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", + "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", + "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" + ], + "version": "==3.4.7" }, "distlib": { "hashes": [ @@ -624,10 +624,10 @@ }, "identify": { "hashes": [ - "sha256:de7129142a5c86d75a52b96f394d94d96d497881d2aaf8eafe320cdbe8ac4bcc", - "sha256:e0dae57c0397629ce13c289f6ddde0204edf518f557bfdb1e56474aa143e77c3" + "sha256:43cb1965e84cdd247e875dec6d13332ef5be355ddc16776396d98089b9053d87", + "sha256:c7c0f590526008911ccc5ceee6ed7b085cbc92f7b6591d0ee5913a130ad64034" ], - "version": "==1.5.14" + "version": "==2.2.2" }, "idna": { "hashes": [ @@ -643,6 +643,13 @@ ], "version": "==1.2.0" }, + "importlib-metadata": { + "hashes": [ + "sha256:c9db46394197244adf2f0b08ec5bc3cf16757e9590b02af1fca085c16c0d600a", + "sha256:d2d46ef77ffc85cbf7dac7e81dd663fde71c45326131bea8033b9bad42268ebe" + ], + "version": "==3.10.0" + }, "jeepney": { "hashes": [ "sha256:7d59b6622675ca9e993a6bd38de845051d315f8b0c72cca3aef733a20b648657", @@ -660,10 +667,10 @@ }, "keyring": { "hashes": [ - "sha256:9acb3e1452edbb7544822b12fd25459078769e560fa51f418b6d00afaa6178df", - "sha256:9f44660a5d4931bdc14c08a1d01ef30b18a7a8147380710d8c9f9531e1f6c3c0" + "sha256:045703609dd3fccfcdb27da201684278823b72af515aedec1a8515719a038cb8", + "sha256:8f607d7d1cc502c43a932a275a56fe47db50271904513a379d39df1af277ac48" ], - "version": "==22.0.1" + "version": "==23.0.1" }, "markupsafe": { "hashes": [ @@ -859,10 +866,10 @@ }, "pygments": { "hashes": [ - "sha256:37a13ba168a02ac54cc5891a42b1caec333e59b66addb7fa633ea8a6d73445c0", - "sha256:b21b072d0ccdf29297a82a2363359d99623597b8a265b8081760e4d0f7153c88" + "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94", + "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "pyparsing": { "hashes": [ @@ -892,15 +899,23 @@ "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", - "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc" + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" ], "version": "==5.4.1" }, @@ -1078,10 +1093,10 @@ }, "tqdm": { "hashes": [ - "sha256:2c44efa73b8914dba7807aefd09653ac63c22b5b4ea34f7a80973f418f1a3089", - "sha256:c23ac707e8e8aabb825e4d91f8e17247f9cc14b0d64dd9e97be0781e9e525bba" + "sha256:daec693491c52e9498632dfbe9ccfc4882a557f5fa08982db1b4d3adbe0887c3", + "sha256:ebdebdb95e3477ceea267decfc0784859aa3df3e27e22d23b83e9b272bf157ae" ], - "version": "==4.58.0" + "version": "==4.60.0" }, "twine": { "hashes": [ @@ -1138,17 +1153,18 @@ }, "urllib3": { "hashes": [ - "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", - "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], - "version": "==1.26.3" + "index": "pypi", + "version": "==1.26.4" }, "virtualenv": { "hashes": [ - "sha256:147b43894e51dd6bba882cf9c282447f780e2251cd35172403745fc381a0a80d", - "sha256:2be72df684b74df0ea47679a7df93fd0e04e72520022c57b479d8f881485dbe3" + "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107", + "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c" ], - "version": "==20.4.2" + "version": "==20.4.3" }, "webencodings": { "hashes": [ @@ -1206,6 +1222,13 @@ "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" ], "version": "==1.6.3" + }, + "zipp": { + "hashes": [ + "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", + "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" + ], + "version": "==3.4.1" } } } From 8e0803e7e5acabdd28b80258f15d8aebf11fbb4c Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Wed, 7 Apr 2021 21:13:11 -0700 Subject: [PATCH 02/81] Add `black` Dockerfile (#1916) * Add `black` Dockerfile - Build a `black` container based on python3-slim - Test: `docker build --network-host --tag black .` ```console cooper-mbp1:black cooper$ docker image ls REPOSITORY TAG IMAGE ID CREATED SIZE black latest 0c7636402ac2 4 seconds ago 332MB ``` Addresses part of #1914 * Build with colorama + d extra installs - Adds ~9mb * Combine all build commands in 1 run - Combine the commands all into 1 RUN to down the size - Get to 65.98 MB image size now: https://hub.docker.com/repository/docker/cooperlees/black/tags?page=1&ordering=last_updated * Add rm -r of /var/lib/apt/lists/* + save 10mb down to 55mb --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..9542479eca5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3-slim + +RUN mkdir /src +COPY . /src/ +RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ + && apt update && apt install -y git \ + && cd /src \ + && pip install --no-cache-dir .[colorama,d] \ + && rm -rf /src \ + && apt remove -y git \ + && apt autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +CMD ["black"] From 4d36304a2c70a5b6e39cdabf404fe72186fc8202 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Apr 2021 10:25:37 -0700 Subject: [PATCH 03/81] Fix error from upcoming typeshed change (#2096) See python/typeshed#5190 --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 431ee027270..54d6edf760a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -847,7 +847,7 @@ async def schedule_formatting( ): src for src in sorted(sources) } - pending: Iterable["asyncio.Future[bool]"] = tasks.keys() + pending = tasks.keys() try: loop.add_signal_handler(signal.SIGINT, cancel, pending) loop.add_signal_handler(signal.SIGTERM, cancel, pending) From e4003c2c439fc08a407d3024a195c010f03c4173 Mon Sep 17 00:00:00 2001 From: johnthagen Date: Sat, 10 Apr 2021 08:07:34 -0400 Subject: [PATCH 04/81] Exclude venv directory by default (#1683) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- CHANGES.md | 2 ++ README.md | 4 ++-- docs/installation_and_usage.md | 4 ++-- src/black/__init__.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 97a3be33c93..6340f60ae59 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,8 @@ - PR #2053: Python 2 support is now optional, install with `python3 -m pip install black[python2]` to maintain support. +- Exclude `venv` directory by default (#1683) + #### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub diff --git a/README.md b/README.md index 0bc0228b7d0..ed9f105f5aa 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,8 @@ Options: directories on all platforms (Windows, too). Exclusions are calculated first, inclusions later. [default: /(\.direnv|\.eggs|\.git|\. - hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_bu - ild|buck-out|build|dist)/] + hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.sv + n|_build|buck-out|build|dist)/] --extend-exclude TEXT Like --exclude, but adds additional files and directories on top of the excluded diff --git a/docs/installation_and_usage.md b/docs/installation_and_usage.md index bb554eb6744..fcde49f4e3a 100644 --- a/docs/installation_and_usage.md +++ b/docs/installation_and_usage.md @@ -87,8 +87,8 @@ Options: directories on all platforms (Windows, too). Exclusions are calculated first, inclusions later. [default: /(\.direnv|\.eggs|\.git|\. - hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_bu - ild|buck-out|build|dist)/] + hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.sv + n|_build|buck-out|build|dist)/] --force-exclude TEXT Like --exclude, but files and directories matching this regex will be excluded even diff --git a/src/black/__init__.py b/src/black/__init__.py index 54d6edf760a..0065beafe0e 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -82,7 +82,7 @@ import colorama # noqa: F401 DEFAULT_LINE_LENGTH = 88 -DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 DEFAULT_INCLUDES = r"\.pyi?$" CACHE_DIR = Path(user_cache_dir("black", version=__version__)) STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" From 2116eca51f1108ebc924185c87bc363d8e0329b3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 11 Apr 2021 07:49:42 -0700 Subject: [PATCH 05/81] fix typing issue around lru_cache arguments (#2098) This was found by python/mypy#10308 --- src/black/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0065beafe0e..2bb42a470ea 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -302,7 +302,7 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) -def find_pyproject_toml(path_search_start: Iterable[str]) -> Optional[str]: +def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: """Find the absolute filepath to a pyproject.toml if it exists""" path_project_root = find_project_root(path_search_start) path_pyproject_toml = path_project_root / "pyproject.toml" @@ -6214,7 +6214,7 @@ def gen_python_files( @lru_cache() -def find_project_root(srcs: Iterable[str]) -> Path: +def find_project_root(srcs: Tuple[str, ...]) -> Path: """Return a directory containing .git, .hg, or pyproject.toml. That directory will be a common parent of all files and directories From ea4e714b9a3b636afd8e81c5c5c916fcf8e6ed42 Mon Sep 17 00:00:00 2001 From: Simon <32608483+J-Exodus@users.noreply.github.com> Date: Mon, 12 Apr 2021 07:00:03 +1000 Subject: [PATCH 06/81] Added not formatting files in gitignore (psf#1682) (#1734) --- docs/pyproject_toml.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pyproject_toml.md b/docs/pyproject_toml.md index 77c12b76be9..ed88f37aa4a 100644 --- a/docs/pyproject_toml.md +++ b/docs/pyproject_toml.md @@ -48,6 +48,8 @@ You can also explicitly specify the path to a particular file that you want with If you're running with `--verbose`, you will see a blue message if a file was found and used. +Files listed within a projects `.gitignore` file will not be formatted by _Black_. + Please note `blackd` will not use `pyproject.toml` configuration. ## Configuration format From d960d5d238bc8c73010a3ca5a9f316678ad91e6f Mon Sep 17 00:00:00 2001 From: Pierre Sassoulas Date: Sun, 11 Apr 2021 23:41:22 +0200 Subject: [PATCH 07/81] Remove NBSP at the beginning of comments (#2092) Closes #2091 --- CHANGES.md | 2 ++ src/black/__init__.py | 7 ++++ tests/data/comments7.py | 7 +++- tests/data/comments_non_breaking_space.py | 44 +++++++++++++++++++++++ tests/test_format.py | 1 + 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/data/comments_non_breaking_space.py diff --git a/CHANGES.md b/CHANGES.md index 6340f60ae59..12e8d453277 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ #### _Black_ +- `Black` now cleans up leading non-breaking spaces in comments (#2092) + - `Black` now respects `--skip-string-normalization` when normalizing multiline docstring quotes (#1637) diff --git a/src/black/__init__.py b/src/black/__init__.py index 2bb42a470ea..36716474e8c 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -2709,6 +2709,13 @@ def make_comment(content: str) -> str: if content[0] == "#": content = content[1:] + NON_BREAKING_SPACE = " " + if ( + content + and content[0] == NON_BREAKING_SPACE + and not content.lstrip().startswith("type:") + ): + content = " " + content[1:] # Replace NBSP by a simple space if content and content[0] not in " !:#'%": content = " " + content return "#" + content diff --git a/tests/data/comments7.py b/tests/data/comments7.py index 0e2bd35bf55..ca9d7c62b21 100644 --- a/tests/data/comments7.py +++ b/tests/data/comments7.py @@ -129,6 +129,8 @@ def test_fails_invalid_post_data( ): ... +square = Square(4) # type: Optional[Square] + # output from .config import ( @@ -263,4 +265,7 @@ class C: def test_fails_invalid_post_data( self, pyramid_config, db_request, post_data, message ): - ... \ No newline at end of file + ... + + +square = Square(4) # type: Optional[Square] diff --git a/tests/data/comments_non_breaking_space.py b/tests/data/comments_non_breaking_space.py new file mode 100644 index 00000000000..e17c3f4ca39 --- /dev/null +++ b/tests/data/comments_non_breaking_space.py @@ -0,0 +1,44 @@ +from .config import ( ConfigTypeAttributes, Int, Path, # String, + # DEFAULT_TYPE_ATTRIBUTES, +) + +result = 1 # A simple comment +result = ( 1, ) # Another one + +result = 1 # type: ignore +result = 1# This comment is talking about type: ignore +square = Square(4) # type: Optional[Square] + +def function(a:int=42): + """ This docstring is already formatted + a + b + """ + #  There's a NBSP + 3 spaces before + # And 4 spaces on the next line + pass + +# output +from .config import ( + ConfigTypeAttributes, + Int, + Path, # String, + # DEFAULT_TYPE_ATTRIBUTES, +) + +result = 1 # A simple comment +result = (1,) # Another one + +result = 1 #  type: ignore +result = 1 # This comment is talking about type: ignore +square = Square(4) #  type: Optional[Square] + + +def function(a: int = 42): + """This docstring is already formatted + a + b + """ + # There's a NBSP + 3 spaces before + # And 4 spaces on the next line + pass diff --git a/tests/test_format.py b/tests/test_format.py index e0cb0b74195..eabec8c7219 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -27,6 +27,7 @@ "comments5", "comments6", "comments7", + "comments_non_breaking_space", "comment_after_escaped_newline", "composition", "composition_no_trailing_comma", From 6d0bdc2f38d57ddf07e7d127215d51f8553997f9 Mon Sep 17 00:00:00 2001 From: Pierre Verkest Date: Mon, 12 Apr 2021 15:18:11 +0200 Subject: [PATCH 08/81] Add missing instructions to make test passed (#2100) --- .github/ISSUE_TEMPLATE/bug_report.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e652f17c94b..f4a438622aa 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -28,7 +28,8 @@ assignees: "" 2. Or run _Black_ on your machine: - create a new virtualenv (make sure it's the same Python version); - clone this repository; - - run `pip install -e .`; + - run `pip install -e .[d,python2]`; + - run `pip install -r test_requirements.txt` - make sure it's sane by running `python -m unittest`; and - run `black` like you did last time. From 9eb29a6d4733c0e467ba5c1c54361f097cc1c305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Tr=C3=A9guier?= Date: Fri, 16 Apr 2021 19:41:19 +0200 Subject: [PATCH 09/81] Fix small comment typo (#2112) We probably don't need to fall back on "polling" when setting up an asyncio loop --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 36716474e8c..bb8f8c7422a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -784,7 +784,7 @@ def reformat_many( except (ImportError, OSError): # we arrive here if the underlying system does not support multi-processing # like in AWS Lambda or Termux, in which case we gracefully fallback to - # a ThreadPollExecutor with just a single worker (more workers would not do us + # a ThreadPoolExecutor with just a single worker (more workers would not do us # any good due to the Global Interpreter Lock) executor = ThreadPoolExecutor(max_workers=1) From 7b4ca4bd90c148d72548048f5109537a555aaf77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Hild=C3=A9n?= Date: Mon, 19 Apr 2021 00:18:39 +0300 Subject: [PATCH 10/81] Instructions on documenting code style modifications (#2109) --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd6ebfca114..f2a4ee20c99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,6 +74,12 @@ To workout X, please use is not perfect but saves a lot of release overhead as now the releaser does not need to go back and workout what to add to the `CHANGES.md` for each release. +### Style Changes + +If a change would affect the advertised code style, please modify the documentation (The +_Black_ code style) to reflect that change. Patches that fix unintended bugs in +formatting don't need to be mentioned separately though. + ### Docs Testing If you make changes to docs, you can test they still build locally too. From 1fc3215e8c8856094b20d497e4e0e3e547ed38eb Mon Sep 17 00:00:00 2001 From: Mark Bell Date: Thu, 22 Apr 2021 16:23:41 +0100 Subject: [PATCH 11/81] Make black remove leading and trailing spaces from one-line docstrings (#1740) Fixes #1738. Fixes #1812. Previously, Black removed leading and trailing spaces in multiline docstrings but failed to remove them from one-line docstrings. --- CHANGES.md | 3 +++ src/black/__init__.py | 40 ++++++++++++++++++++++++------------ src/black_primer/primer.json | 10 ++++----- tests/data/docstring.py | 36 +++++++++++++++++++++++++++++++- tox.ini | 2 +- 5 files changed, 71 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 12e8d453277..f911fdf77ae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ #### _Black_ +- `Black` now processes one-line docstrings by stripping leading and trailing spaces, + and adding a padding space when needed to break up """". (#1740) + - `Black` now cleans up leading non-breaking spaces in comments (#2092) - `Black` now respects `--skip-string-normalization` when normalizing multiline diff --git a/src/black/__init__.py b/src/black/__init__.py index bb8f8c7422a..efa82f41160 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -2153,16 +2153,35 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. prefix = get_string_prefix(leaf.value) - lead_len = len(prefix) + 3 - tail_len = -3 - indent = " " * 4 * self.current_line.depth - docstring = fix_docstring(leaf.value[lead_len:tail_len], indent) + docstring = leaf.value[len(prefix) :] # Remove the prefix + quote_char = docstring[0] + # A natural way to remove the outer quotes is to do: + # docstring = docstring.strip(quote_char) + # but that breaks on """""x""" (which is '""x'). + # So we actually need to remove the first character and the next two + # characters but only if they are the same as the first. + quote_len = 1 if docstring[1] != quote_char else 3 + docstring = docstring[quote_len:-quote_len] + + if is_multiline_string(leaf): + indent = " " * 4 * self.current_line.depth + docstring = fix_docstring(docstring, indent) + else: + docstring = docstring.strip() + if docstring: - if leaf.value[lead_len - 1] == docstring[0]: + # Add some padding if the docstring starts / ends with a quote mark. + if docstring[0] == quote_char: docstring = " " + docstring - if leaf.value[tail_len + 1] == docstring[-1]: + if docstring[-1] == quote_char: docstring = docstring + " " - leaf.value = leaf.value[0:lead_len] + docstring + leaf.value[tail_len:] + else: + # Add some padding if the docstring is empty. + docstring = " " + + # We could enforce triple quotes at this point. + quote = quote_char * quote_len + leaf.value = prefix + quote + docstring + quote yield from self.visit_default(leaf) @@ -6113,7 +6132,7 @@ def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]: @lru_cache() def get_gitignore(root: Path) -> PathSpec: - """ Return a PathSpec matching gitignore content if present.""" + """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): @@ -6953,11 +6972,6 @@ def patched_main() -> None: def is_docstring(leaf: Leaf) -> bool: - if not is_multiline_string(leaf): - # For the purposes of docstring re-indentation, we don't need to do anything - # with single-line docstrings. - return False - if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 32df01571a7..3a1fae090a0 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -3,7 +3,7 @@ "projects": { "aioexabgp": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] @@ -17,7 +17,7 @@ }, "bandersnatch": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] @@ -84,7 +84,7 @@ }, "ptr": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/facebookincubator/ptr.git", "long_checkout": false, "py_versions": ["all"] @@ -98,14 +98,14 @@ }, "sqlalchemy": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git", "long_checkout": false, "py_versions": ["all"] }, "tox": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/tox-dev/tox.git", "long_checkout": false, "py_versions": ["all"] diff --git a/tests/data/docstring.py b/tests/data/docstring.py index 5c6985d0e08..74532b2b91d 100644 --- a/tests/data/docstring.py +++ b/tests/data/docstring.py @@ -102,6 +102,23 @@ def and_this(): "hey yah"''' +def empty(): + ''' + + + + + ''' + + +def oneline_empty(): + ''' ''' + + +def single_quotes(): + 'testing' + + def believe_it_or_not_this_is_in_the_py_stdlib(): ''' "hey yah"''' @@ -110,6 +127,8 @@ def ignored_docstring(): """a => \ b""" +def single_line_docstring_with_whitespace(): + """ This should be stripped """ def docstring_with_inline_tabs_and_space_indentation(): """hey @@ -134,7 +153,6 @@ def docstring_with_inline_tabs_and_tab_indentation(): line ends with some tabs """ pass - # output @@ -241,6 +259,18 @@ def and_this(): "hey yah"''' +def empty(): + """ """ + + +def oneline_empty(): + """ """ + + +def single_quotes(): + "testing" + + def believe_it_or_not_this_is_in_the_py_stdlib(): ''' "hey yah"''' @@ -251,6 +281,10 @@ def ignored_docstring(): b""" +def single_line_docstring_with_whitespace(): + """This should be stripped""" + + def docstring_with_inline_tabs_and_space_indentation(): """hey diff --git a/tox.ini b/tox.ini index 9bb809abe41..71b02e920c7 100644 --- a/tox.ini +++ b/tox.ini @@ -23,4 +23,4 @@ commands = pip install -e .[d] coverage erase coverage run fuzz.py - coverage report \ No newline at end of file + coverage report From 5316bbff0e9db989db98b14761ce2c299a05895b Mon Sep 17 00:00:00 2001 From: CiderMan Date: Thu, 22 Apr 2021 16:40:51 +0100 Subject: [PATCH 12/81] Handle Docstrings as bytes + strip all whitespace (#2037) (fixes #1844, fixes #1923, fixes #1851, fixes #2002, fixes #2103) --- CHANGES.md | 3 +++ src/black/__init__.py | 14 ++++++++++++-- tests/test_black.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f911fdf77ae..2999abfff4e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,6 +52,9 @@ - Exclude `venv` directory by default (#1683) +- Fixed "Black produced code that is not equivalent to the source" when formatting + Python 2 docstrings (#2037) + #### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub diff --git a/src/black/__init__.py b/src/black/__init__.py index efa82f41160..5c9ab751650 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -6466,12 +6466,22 @@ def _stringify_ast( # Constant strings may be indented across newlines, if they are # docstrings; fold spaces after newlines when comparing. Similarly, # trailing and leading space may be removed. + # Note that when formatting Python 2 code, at least with Windows + # line-endings, docstrings can end up here as bytes instead of + # str so make sure that we handle both cases. if ( isinstance(node, ast.Constant) and field == "value" - and isinstance(value, str) + and isinstance(value, (str, bytes)) ): - normalized = re.sub(r" *\n[ \t]*", "\n", value).strip() + lineend = "\n" if isinstance(value, str) else b"\n" + # To normalize, we strip any leading and trailing space from + # each line... + stripped = [line.strip() for line in value.splitlines()] + normalized = lineend.join(stripped) # type: ignore[attr-defined] + # ...and remove any blank lines at the beginning and end of + # the whole string + normalized = normalized.strip() else: normalized = value yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" diff --git a/tests/test_black.py b/tests/test_black.py index c603233efc4..201edfa2de5 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1918,6 +1918,26 @@ def test_bpo_2142_workaround(self) -> None: actual = diff_header.sub(DETERMINISTIC_HEADER, actual) self.assertEqual(actual, expected) + def test_docstring_reformat_for_py27(self) -> None: + """ + Check that stripping trailing whitespace from Python 2 docstrings + doesn't trigger a "not equivalent to source" error + """ + source = ( + b'def foo():\r\n """Testing\r\n Testing """\r\n print "Foo"\r\n' + ) + expected = 'def foo():\n """Testing\n Testing"""\n print "Foo"\n' + + result = CliRunner().invoke( + black.main, + ["-", "-q", "--target-version=py27"], + input=BytesIO(source), + ) + + self.assertEqual(result.exit_code, 0) + actual = result.output + self.assertFormatEqual(actual, expected) + with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() From 368f043f138112f63ff521c3481993c589eb7508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Hild=C3=A9n?= Date: Thu, 22 Apr 2021 20:37:27 +0300 Subject: [PATCH 13/81] Document experimental string processing and docstring indentation (#2106) --- docs/the_black_code_style.md | 15 +++++++++++++++ tests/data/docstring.py | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/the_black_code_style.md b/docs/the_black_code_style.md index 1cc591b8031..67ccb8aab11 100644 --- a/docs/the_black_code_style.md +++ b/docs/the_black_code_style.md @@ -289,6 +289,21 @@ If you are adopting _Black_ in a large project with pre-existing string conventi you can pass `--skip-string-normalization` on the command line. This is meant as an adoption helper, avoid using this for new projects. +As an experimental option, _Black_ splits long strings (using parentheses where +appropriate) and merges short ones. When split, parts of f-strings that don't need +formatting are converted to plain strings. User-made splits are respected when they do +not exceed the line length limit. Line continuation backslashes are converted into +parenthesized strings. Unnecessary parentheses are stripped. To enable experimental +string processing, pass `--experimental-string-processing` on the command line. Because +the functionality is experimental, feedback and issue reports are highly encouraged! + +_Black_ also processes docstrings. Firstly the indentation of docstrings is corrected +for both quotations and the text within, although relative indentation in the text is +preserved. Superfluous trailing whitespace on each line and unnecessary new lines at the +end of the docstring are removed. All leading tabs are converted to spaces, but tabs +inside text are preserved. Whitespace leading and trailing one-line docstrings is +removed. The quotations of an empty docstring are separated with one space. + ### Numeric literals _Black_ standardizes most numeric literals to use lowercase letters for the syntactic diff --git a/tests/data/docstring.py b/tests/data/docstring.py index 74532b2b91d..9e1c2441e0e 100644 --- a/tests/data/docstring.py +++ b/tests/data/docstring.py @@ -115,6 +115,10 @@ def oneline_empty(): ''' ''' +def oneline_nothing(): + """""" + + def single_quotes(): 'testing' @@ -267,6 +271,10 @@ def oneline_empty(): """ """ +def oneline_nothing(): + """ """ + + def single_quotes(): "testing" From 773e4a22d58be1f1aa82df7ad9a0f4c4e1328c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 25 Apr 2021 19:04:15 +0200 Subject: [PATCH 14/81] Revert "Use lowercase hex numbers fixes #1692 (#1775)" This reverts commit 7d032fa848c8910007a0a41c1ba61d70d2846f48. --- .gitignore | 3 +-- src/black/__init__.py | 9 ++------- src/black_primer/primer.json | 10 +++++----- src/blib2to3/pytree.py | 2 +- tests/data/numeric_literals.py | 4 ++-- tests/data/numeric_literals_py2.py | 4 ++-- tests/data/numeric_literals_skip_underscores.py | 6 +++--- 7 files changed, 16 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 30225ec7764..3207e72ae28 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.venv .coverage _build .DS_Store @@ -16,4 +15,4 @@ src/_black_version.py .dmypy.json *.swp .hypothesis/ -venv/ +venv/ \ No newline at end of file diff --git a/src/black/__init__.py b/src/black/__init__.py index 5c9ab751650..0a893aa80fe 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5351,15 +5351,10 @@ def normalize_numeric_literal(leaf: Leaf) -> None: def format_hex(text: str) -> str: """ - Formats a hexadecimal string like "0x12b3" - - Uses lowercase because of similarity between "B" and "8", which - can cause security issues. - see: https://github.com/psf/black/issues/1692 + Formats a hexadecimal string like "0x12B3" """ - before, after = text[:2], text[2:] - return f"{before}{after.lower()}" + return f"{before}{after.upper()}" def format_scientific_notation(text: str) -> str: diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 3a1fae090a0..7f7c3b0f37b 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -10,7 +10,7 @@ }, "attrs": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/python-attrs/attrs.git", "long_checkout": false, "py_versions": ["all"] @@ -47,7 +47,7 @@ }, "hypothesis": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/HypothesisWorks/hypothesis.git", "long_checkout": false, "py_versions": ["all"] @@ -63,7 +63,7 @@ }, "pillow": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/python-pillow/Pillow.git", "long_checkout": false, "py_versions": ["all"] @@ -77,7 +77,7 @@ }, "pyramid": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/Pylons/pyramid.git", "long_checkout": false, "py_versions": ["all"] @@ -112,7 +112,7 @@ }, "virtualenv": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/pypa/virtualenv.git", "long_checkout": false, "py_versions": ["all"] diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index 0c074f6a4ac..7843467e012 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -33,7 +33,7 @@ import sys from io import StringIO -HUGE: int = 0x7fffffff # maximum repeat count, default max +HUGE: int = 0x7FFFFFFF # maximum repeat count, default max _type_reprs: Dict[int, Union[Text, int]] = {} diff --git a/tests/data/numeric_literals.py b/tests/data/numeric_literals.py index 06b7f7758ee..254da68d330 100644 --- a/tests/data/numeric_literals.py +++ b/tests/data/numeric_literals.py @@ -12,7 +12,7 @@ x = 123456789E123456789 x = 123456789J x = 123456789.123456789J -x = 0Xb1aCc +x = 0XB1ACC x = 0B1011 x = 0O777 x = 0.000000006 @@ -36,7 +36,7 @@ x = 123456789e123456789 x = 123456789j x = 123456789.123456789j -x = 0xb1acc +x = 0xB1ACC x = 0b1011 x = 0o777 x = 0.000000006 diff --git a/tests/data/numeric_literals_py2.py b/tests/data/numeric_literals_py2.py index 8b2c7faa306..8f85c43f265 100644 --- a/tests/data/numeric_literals_py2.py +++ b/tests/data/numeric_literals_py2.py @@ -3,7 +3,7 @@ x = 123456789L x = 123456789l x = 123456789 -x = 0xB1aCc +x = 0xb1acc # output @@ -13,4 +13,4 @@ x = 123456789L x = 123456789L x = 123456789 -x = 0xb1acc +x = 0xB1ACC diff --git a/tests/data/numeric_literals_skip_underscores.py b/tests/data/numeric_literals_skip_underscores.py index f83e23312f2..e345bb90276 100644 --- a/tests/data/numeric_literals_skip_underscores.py +++ b/tests/data/numeric_literals_skip_underscores.py @@ -3,7 +3,7 @@ x = 123456789 x = 1_2_3_4_5_6_7 x = 1E+1 -x = 0xb1AcC +x = 0xb1acc x = 0.00_00_006 x = 12_34_567J x = .1_2 @@ -16,8 +16,8 @@ x = 123456789 x = 1_2_3_4_5_6_7 x = 1e1 -x = 0xb1acc +x = 0xB1ACC x = 0.00_00_006 x = 12_34_567j x = 0.1_2 -x = 1_2.0 +x = 1_2.0 \ No newline at end of file From ea9fefb359b86208ad20c944c3c420e6c3fb2c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 25 Apr 2021 19:42:02 +0200 Subject: [PATCH 15/81] Re-add .venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3207e72ae28..f487d96cb3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.venv .coverage _build .DS_Store From 5e09fa07f5db113c9b0858da15eff6bb31fe2417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 25 Apr 2021 19:44:06 +0200 Subject: [PATCH 16/81] Fix primer config --- src/black_primer/primer.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 7f7c3b0f37b..1dd380b3cb4 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -10,7 +10,7 @@ }, "attrs": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-attrs/attrs.git", "long_checkout": false, "py_versions": ["all"] @@ -63,7 +63,7 @@ }, "pillow": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-pillow/Pillow.git", "long_checkout": false, "py_versions": ["all"] @@ -77,7 +77,7 @@ }, "pyramid": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/Pylons/pyramid.git", "long_checkout": false, "py_versions": ["all"] @@ -91,7 +91,7 @@ }, "pytest": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/pytest-dev/pytest.git", "long_checkout": false, "py_versions": ["all"] @@ -112,7 +112,7 @@ }, "virtualenv": { "cli_arguments": [], - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/virtualenv.git", "long_checkout": false, "py_versions": ["all"] From 8672af35f052a636545e38110f0419ea92aeca0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 25 Apr 2021 20:15:54 +0200 Subject: [PATCH 17/81] Work around stability errors due to optional trailing commas (#2126) Optional trailing commas put by Black become magic trailing commas on another pass of the tool. Since they are influencing formatting around optional parentheses, on rare occasions the tool changes its mind in terms of putting parentheses or not. Ideally this would never be the case but sadly the decision to put optional parentheses or not (which looks at pre-existing "magic" trailing commas) is happening around the same time as the decision to put an optional trailing comma. Untangling the two proved to be impractically difficult. This shameful workaround uses the fact that the formatting instability introduced by magic trailing commas is deterministic: if the optional trailing comma becoming a pre-existing "magic" trailing comma changes formatting, the second pass becomes stable since there is no variable factor anymore on pass 3, 4, and so on. For most files, this will introduce no performance penalty since `--safe` is already re-formatting everything twice to ensure formatting stability. We're using this result and if all's good, the behavior is equivalent. If there is a difference, we treat the second result as the binding one, and check its sanity again. --- CHANGES.md | 4 ++++ src/black/__init__.py | 24 +++++++++++++++++------- tests/test_black.py | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2999abfff4e..0ca0b84f6d1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,10 @@ #### _Black_ +- Fixed a rare but annoying formatting instability created by the combination of + optional trailing commas inserted by `Black` and optional parentheses looking at + pre-existing "magic" trailing commas (#2126) + - `Black` now processes one-line docstrings by stripping leading and trailing spaces, and adding a padding space when needed to break up """". (#1740) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0a893aa80fe..08267d50f89 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1016,7 +1016,17 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if not fast: assert_equivalent(src_contents, dst_contents) - assert_stable(src_contents, dst_contents, mode=mode) + + # Forced second pass to work around optional trailing commas (becoming + # forced trailing commas on pass 2) interacting differently with optional + # parentheses. Admittedly ugly. + dst_contents_pass2 = format_str(dst_contents, mode=mode) + if dst_contents != dst_contents_pass2: + dst_contents = dst_contents_pass2 + assert_equivalent(src_contents, dst_contents, pass_num=2) + assert_stable(src_contents, dst_contents, mode=mode) + # Note: no need to explicitly call `assert_stable` if `dst_contents` was + # the same as `dst_contents_pass2`. return dst_contents @@ -6484,7 +6494,7 @@ def _stringify_ast( yield f"{' ' * depth}) # /{node.__class__.__name__}" -def assert_equivalent(src: str, dst: str) -> None: +def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None: """Raise AssertionError if `src` and `dst` aren't equivalent.""" try: src_ast = parse_ast(src) @@ -6499,9 +6509,9 @@ def assert_equivalent(src: str, dst: str) -> None: except Exception as exc: log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst) raise AssertionError( - f"INTERNAL ERROR: Black produced invalid code: {exc}. Please report a bug" - " on https://github.com/psf/black/issues. This invalid output might be" - f" helpful: {log}" + f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. " + "Please report a bug on https://github.com/psf/black/issues. " + f"This invalid output might be helpful: {log}" ) from None src_ast_str = "\n".join(_stringify_ast(src_ast)) @@ -6510,8 +6520,8 @@ def assert_equivalent(src: str, dst: str) -> None: log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst")) raise AssertionError( "INTERNAL ERROR: Black produced code that is not equivalent to the" - " source. Please report a bug on https://github.com/psf/black/issues. " - f" This diff might be helpful: {log}" + f" source on pass {pass_num}. Please report a bug on " + f"https://github.com/psf/black/issues. This diff might be helpful: {log}" ) from None diff --git a/tests/test_black.py b/tests/test_black.py index 201edfa2de5..8733b7a63ad 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -254,6 +254,24 @@ def test_trailing_comma_optional_parens_stability3(self) -> None: actual = fs(source) black.assert_stable(source, actual, DEFAULT_MODE) + @patch("black.dump_to_file", dump_to_stderr) + def test_trailing_comma_optional_parens_stability1_pass2(self) -> None: + source, _expected = read_data("trailing_comma_optional_parens1") + actual = fs(fs(source)) # this is what `format_file_contents` does with --safe + black.assert_stable(source, actual, DEFAULT_MODE) + + @patch("black.dump_to_file", dump_to_stderr) + def test_trailing_comma_optional_parens_stability2_pass2(self) -> None: + source, _expected = read_data("trailing_comma_optional_parens2") + actual = fs(fs(source)) # this is what `format_file_contents` does with --safe + black.assert_stable(source, actual, DEFAULT_MODE) + + @patch("black.dump_to_file", dump_to_stderr) + def test_trailing_comma_optional_parens_stability3_pass2(self) -> None: + source, _expected = read_data("trailing_comma_optional_parens3") + actual = fs(fs(source)) # this is what `format_file_contents` does with --safe + black.assert_stable(source, actual, DEFAULT_MODE) + @patch("black.dump_to_file", dump_to_stderr) def test_pep_572(self) -> None: source, expected = read_data("pep_572") From e5490e96232e301e9eed971984bb75f2f87475b0 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 25 Apr 2021 11:27:35 -0700 Subject: [PATCH 18/81] Add Docker Github Action (#2086) * Add Docker Github Action - Build and upload arm64 + amd64 black images on push to master This will need a `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` secrets set by someone with access. * Change tag to push to pyfound/black repository. Thanks @ewdurbin --- .github/workflows/docker.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..40b3d07736b --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,33 @@ +name: docker + +on: + push: + branches: + - "master" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: pyfound/black:latest + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} From eee949e286363384b25426b9ce461d31a5842ebf Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 25 Apr 2021 14:51:13 -0700 Subject: [PATCH 19/81] Docker CI: Add missed Checkout step (#2128) - Reading my error others hit it by forgetting this Checkoutstep too so trying the fix - e.g. https://github.com/docker/build-push-action/issues/179 - Makes sense it's needed --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 40b3d07736b..09390130b3d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,9 @@ jobs: docker: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU uses: docker/setup-qemu-action@v1 From de7187a823bd701ecb75c62bca0d7bd00076108a Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sun, 25 Apr 2021 17:52:23 -0400 Subject: [PATCH 20/81] Issue 1629 has been closed, let's celebrate! (#2127) --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0ca0b84f6d1..65d037394ad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,8 @@ - Fixed a rare but annoying formatting instability created by the combination of optional trailing commas inserted by `Black` and optional parentheses looking at - pre-existing "magic" trailing commas (#2126) + pre-existing "magic" trailing commas. This fixes issue #1629 and all of its many many + duplicates. (#2126) - `Black` now processes one-line docstrings by stripping leading and trailing spaces, and adding a padding space when needed to break up """". (#1740) From 67d5532c3392280de0ce717a1ab728eca2beb698 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 25 Apr 2021 15:12:27 -0700 Subject: [PATCH 21/81] Update CHANGES.md for release (#2129) --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 65d037394ad..76653853d90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ ## Change Log -### Unreleased +### 21.4b0 #### _Black_ From 65f0ea61599dc6d2be95ee961c42a55247130c49 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 25 Apr 2021 20:48:04 -0700 Subject: [PATCH 22/81] Remove Lowercase Hex (PR #1692) from CHANGES.md (#2133) - It was reverted to not cause so much diff churn on millions of lines of code - Fix primer config for projects that should now pass --- CHANGES.md | 2 -- src/black_primer/primer.json | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 76653853d90..e313a44474d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -37,8 +37,6 @@ - Added parsing support for unparenthesized tuples and yield expressions in annotated assignments (#1835) -- use lowercase hex strings (#1692) - - added `--extend-exclude` argument (PR #2005) - speed up caching by avoiding pathlib (#1950) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 1dd380b3cb4..7a6f5abf408 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -3,7 +3,7 @@ "projects": { "aioexabgp": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] @@ -17,7 +17,7 @@ }, "bandersnatch": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] From 0a833b4b14953f98e81d632281a75318faa66170 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 25 Apr 2021 22:46:48 -0700 Subject: [PATCH 23/81] fix magic comma and experimental string cache flags (#2131) * fix magic comma and experimental string cache flags * more changelog * Update CHANGES.md Co-authored-by: Cooper Lees * fix tests Co-authored-by: Cooper Lees --- CHANGES.md | 8 ++++++++ src/black/__init__.py | 4 +++- tests/test_black.py | 7 ++++--- tests/test_format.py | 37 ++++++++++++++++++++++--------------- tests/util.py | 2 +- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e313a44474d..aa08396e491 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,13 @@ ## Change Log +### Unreleased + +#### _Black_ + +- Reflect the `--skip-magic-trailing-comma` and `--experimental-string-processing` flags + in the name of the cache file. Without this fix, changes in these flags would not take + effect if the cache had already been populated. (#2131) + ### 21.4b0 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 08267d50f89..00d3729c835 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -273,9 +273,9 @@ class Mode: target_versions: Set[TargetVersion] = field(default_factory=set) line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True + is_pyi: bool = False magic_trailing_comma: bool = True experimental_string_processing: bool = False - is_pyi: bool = False def get_cache_key(self) -> str: if self.target_versions: @@ -290,6 +290,8 @@ def get_cache_key(self) -> str: str(self.line_length), str(int(self.string_normalization)), str(int(self.is_pyi)), + str(int(self.magic_trailing_comma)), + str(int(self.experimental_string_processing)), ] return ".".join(parts) diff --git a/tests/test_black.py b/tests/test_black.py index 8733b7a63ad..88ea31777fb 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -379,11 +379,12 @@ def test_detect_pos_only_arguments(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: source, expected = read_data("string_quotes") - actual = fs(source) + mode = black.Mode(experimental_string_processing=True) + actual = fs(source, mode=mode) self.assertFormatEqual(expected, actual) black.assert_equivalent(source, actual) - black.assert_stable(source, actual, DEFAULT_MODE) - mode = replace(DEFAULT_MODE, string_normalization=False) + black.assert_stable(source, actual, mode) + mode = replace(mode, string_normalization=False) not_normalized = fs(source, mode=mode) self.assertFormatEqual(source.replace("\\\n", ""), not_normalized) black.assert_equivalent(source, not_normalized) diff --git a/tests/test_format.py b/tests/test_format.py index eabec8c7219..e1335aedf43 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -16,7 +16,6 @@ SIMPLE_CASES = [ "beginning_backslash", "bracketmatch", - "cantfit", "class_blank_parentheses", "class_methods_new_line", "collections", @@ -26,7 +25,6 @@ "comments4", "comments5", "comments6", - "comments7", "comments_non_breaking_space", "comment_after_escaped_newline", "composition", @@ -48,11 +46,7 @@ "function2", "function_trailing_comma", "import_spacing", - "long_strings", - "long_strings__edge_case", - "long_strings__regression", "numeric_literals_py2", - "percent_precedence", "python2", "python2_unicode_literals", "remove_parens", @@ -62,6 +56,15 @@ "tupleassign", ] +EXPERIMENTAL_STRING_PROCESSING_CASES = [ + "cantfit", + "comments7", + "long_strings", + "long_strings__edge_case", + "long_strings__regression", + "percent_precedence", +] + SOURCES = [ "tests/test_black.py", @@ -86,19 +89,23 @@ class TestSimpleFormat(BlackBaseTestCase): @parameterized.expand(SIMPLE_CASES) @patch("black.dump_to_file", dump_to_stderr) def test_simple_format(self, filename: str) -> None: - source, expected = read_data(filename) - actual = fs(source) - self.assertFormatEqual(expected, actual) - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, DEFAULT_MODE) + self.check_file(filename, DEFAULT_MODE) + + @parameterized.expand(EXPERIMENTAL_STRING_PROCESSING_CASES) + @patch("black.dump_to_file", dump_to_stderr) + def test_experimental_format(self, filename: str) -> None: + self.check_file(filename, black.Mode(experimental_string_processing=True)) @parameterized.expand(SOURCES) @patch("black.dump_to_file", dump_to_stderr) def test_source_is_formatted(self, filename: str) -> None: path = THIS_DIR.parent / filename - source, expected = read_data(str(path), data=False) - actual = fs(source, mode=DEFAULT_MODE) + self.check_file(str(path), DEFAULT_MODE, data=False) + self.assertFalse(ff(path)) + + def check_file(self, filename: str, mode: black.Mode, *, data: bool = True) -> None: + source, expected = read_data(filename, data=data) + actual = fs(source, mode=mode) self.assertFormatEqual(expected, actual) black.assert_equivalent(source, actual) - black.assert_stable(source, actual, DEFAULT_MODE) - self.assertFalse(ff(path)) + black.assert_stable(source, actual, mode) diff --git a/tests/util.py b/tests/util.py index da65ed0cc93..ad9866975ac 100644 --- a/tests/util.py +++ b/tests/util.py @@ -12,7 +12,7 @@ DETERMINISTIC_HEADER = "[Deterministic header]" -DEFAULT_MODE = black.FileMode(experimental_string_processing=True) +DEFAULT_MODE = black.Mode() ff = partial(black.format_file_in_place, mode=DEFAULT_MODE, fast=True) fs = partial(black.format_str, mode=DEFAULT_MODE) From db30456916ca733d011a2774e66beb3dd7064e14 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 26 Apr 2021 11:28:42 -0400 Subject: [PATCH 24/81] Don't strip parens in assert / return with assign expr (#2143) Black would previously strip the parenthesis away from statements like this these ones: assert (spam := 12 + 1) return (cheese := 1 - 12) Which happens to be invalid code. Now before making the parenthesis invisible, Black checks if the assignment expression's parent is an assert stamtment, aborting if True. Raise, yield, and await are already handled fine. I added a bunch of test cases from the PEP defining asssignment expressions (PEP 572). --- CHANGES.md | 3 ++ src/black/__init__.py | 7 +++- tests/data/pep_572_remove_parens.py | 52 +++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index aa08396e491..9fe6b7e3fa4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ in the name of the cache file. Without this fix, changes in these flags would not take effect if the cache had already been populated. (#2131) +- Don't remove necessary parentheses from assignment expression containing assert / + return statements. (#2143) + ### 21.4b0 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 00d3729c835..99afc7d3c50 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5613,7 +5613,12 @@ def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: return False if is_walrus_assignment(node): - if parent.type in [syms.annassign, syms.expr_stmt]: + if parent.type in [ + syms.annassign, + syms.expr_stmt, + syms.assert_stmt, + syms.return_stmt, + ]: return False first = node.children[0] diff --git a/tests/data/pep_572_remove_parens.py b/tests/data/pep_572_remove_parens.py index 04cc75bc36c..9a804859a09 100644 --- a/tests/data/pep_572_remove_parens.py +++ b/tests/data/pep_572_remove_parens.py @@ -17,6 +17,32 @@ a, b = (test := (1, 2)) +# see also https://github.com/psf/black/issues/2139 +assert (foo := 42 - 12) + +foo(x=(y := f(x))) + + +def foo(answer=(p := 42)): + ... + + +def foo2(answer: (p := 42) = 5): + ... + + +lambda: (x := 1) + +# we don't touch expressions in f-strings but if we do one day, don't break 'em +f'{(x:=10)}' + + +def a(): + return (x := 3) + await (b := 1) + yield (a := 2) + raise (c := 3) + # output if foo := 0: pass @@ -36,3 +62,29 @@ test: int = (test2 := 2) a, b = (test := (1, 2)) + +# see also https://github.com/psf/black/issues/2139 +assert (foo := 42 - 12) + +foo(x=(y := f(x))) + + +def foo(answer=(p := 42)): + ... + + +def foo2(answer: (p := 42) = 5): + ... + + +lambda: (x := 1) + +# we don't touch expressions in f-strings but if we do one day, don't break 'em +f"{(x:=10)}" + + +def a(): + return (x := 3) + await (b := 1) + yield (a := 2) + raise (c := 3) From 97c24664c5bc251f7b67756b8b184962ca44b6d9 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 26 Apr 2021 13:26:03 -0400 Subject: [PATCH 25/81] Symlink docs/change_log.md to CHANGES.md, don't copy (#2146) Super duper janky stopgap fix until I get my documentation reorganization work done and merged --- docs/change_log.md | 537 +-------------------------------------------- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 537 deletions(-) mode change 100644 => 120000 docs/change_log.md diff --git a/docs/change_log.md b/docs/change_log.md deleted file mode 100644 index 01c27553fca..00000000000 --- a/docs/change_log.md +++ /dev/null @@ -1,536 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM CHANGES.md" - -## Change Log - -### Unreleased - -#### _Black_ - -- `Black` now respects `--skip-string-normalization` when normalizing multiline - docstring quotes (#1637) - -- `Black` no longer removes all empty lines between non-function code and decorators - when formatting typing stubs. Now `Black` enforces a single empty line. (#1646) - -- `Black` no longer adds an incorrect space after a parenthesized assignment expression - in if/while statements (#1655) - -- fixed a crash when PWD=/ on POSIX (#1631) - -- fixed "I/O operation on closed file" when using --diff (#1664) - -- Prevent coloured diff output being interleaved with multiple files (#1673) - -- Added support for PEP 614 relaxed decorator syntax on python 3.9 (#1711) - -- Added parsing support for unparenthesized tuples and yield expressions in annotated - assignments (#1835) - -- use lowercase hex strings (#1692) - -- added `--extend-exclude` argument (#1571) - -#### _Packaging_ - -- Self-contained native _Black_ binaries are now provided for releases via GitHub - Releases (#1743) - -### 20.8b1 - -#### _Packaging_ - -- explicitly depend on Click 7.1.2 or newer as `Black` no longer works with versions - older than 7.0 - -### 20.8b0 - -#### _Black_ - -- re-implemented support for explicit trailing commas: now it works consistently within - any bracket pair, including nested structures (#1288 and duplicates) - -- `Black` now reindents docstrings when reindenting code around it (#1053) - -- `Black` now shows colored diffs (#1266) - -- `Black` is now packaged using 'py3' tagged wheels (#1388) - -- `Black` now supports Python 3.8 code, e.g. star expressions in return statements - (#1121) - -- `Black` no longer normalizes capital R-string prefixes as those have a - community-accepted meaning (#1244) - -- `Black` now uses exit code 2 when specified configuration file doesn't exit (#1361) - -- `Black` now works on AWS Lambda (#1141) - -- added `--force-exclude` argument (#1032) - -- removed deprecated `--py36` option (#1236) - -- fixed `--diff` output when EOF is encountered (#526) - -- fixed `# fmt: off` handling around decorators (#560) - -- fixed unstable formatting with some `# type: ignore` comments (#1113) - -- fixed invalid removal on organizing brackets followed by indexing (#1575) - -- introduced `black-primer`, a CI tool that allows us to run regression tests against - existing open source users of Black (#1402) - -- introduced property-based fuzzing to our test suite based on Hypothesis and - Hypothersmith (#1566) - -- implemented experimental and disabled by default long string rewrapping (#1132), - hidden under a `--experimental-string-processing` flag while it's being worked on; - this is an undocumented and unsupported feature, you lose Internet points for - depending on it (#1609) - -#### Vim plugin - -- prefer virtualenv packages over global packages (#1383) - -### 19.10b0 - -- added support for PEP 572 assignment expressions (#711) - -- added support for PEP 570 positional-only arguments (#943) - -- added support for async generators (#593) - -- added support for pre-splitting collections by putting an explicit trailing comma - inside (#826) - -- added `black -c` as a way to format code passed from the command line (#761) - -- --safe now works with Python 2 code (#840) - -- fixed grammar selection for Python 2-specific code (#765) - -- fixed feature detection for trailing commas in function definitions and call sites - (#763) - -- `# fmt: off`/`# fmt: on` comment pairs placed multiple times within the same block of - code now behave correctly (#1005) - -- _Black_ no longer crashes on Windows machines with more than 61 cores (#838) - -- _Black_ no longer crashes on standalone comments prepended with a backslash (#767) - -- _Black_ no longer crashes on `from` ... `import` blocks with comments (#829) - -- _Black_ no longer crashes on Python 3.7 on some platform configurations (#494) - -- _Black_ no longer fails on comments in from-imports (#671) - -- _Black_ no longer fails when the file starts with a backslash (#922) - -- _Black_ no longer merges regular comments with type comments (#1027) - -- _Black_ no longer splits long lines that contain type comments (#997) - -- removed unnecessary parentheses around `yield` expressions (#834) - -- added parentheses around long tuples in unpacking assignments (#832) - -- added parentheses around complex powers when they are prefixed by a unary operator - (#646) - -- fixed bug that led _Black_ format some code with a line length target of 1 (#762) - -- _Black_ no longer introduces quotes in f-string subexpressions on string boundaries - (#863) - -- if _Black_ puts parenthesis around a single expression, it moves comments to the - wrapped expression instead of after the brackets (#872) - -- `blackd` now returns the version of _Black_ in the response headers (#1013) - -- `blackd` can now output the diff of formats on source code when the `X-Diff` header is - provided (#969) - -### 19.3b0 - -- new option `--target-version` to control which Python versions _Black_-formatted code - should target (#618) - -- deprecated `--py36` (use `--target-version=py36` instead) (#724) - -- _Black_ no longer normalizes numeric literals to include `_` separators (#696) - -- long `del` statements are now split into multiple lines (#698) - -- type comments are no longer mangled in function signatures - -- improved performance of formatting deeply nested data structures (#509) - -- _Black_ now properly formats multiple files in parallel on Windows (#632) - -- _Black_ now creates cache files atomically which allows it to be used in parallel - pipelines (like `xargs -P8`) (#673) - -- _Black_ now correctly indents comments in files that were previously formatted with - tabs (#262) - -- `blackd` now supports CORS (#622) - -### 18.9b0 - -- numeric literals are now formatted by _Black_ (#452, #461, #464, #469): - - - numeric literals are normalized to include `_` separators on Python 3.6+ code - - - added `--skip-numeric-underscore-normalization` to disable the above behavior and - leave numeric underscores as they were in the input - - - code with `_` in numeric literals is recognized as Python 3.6+ - - - most letters in numeric literals are lowercased (e.g., in `1e10`, `0x01`) - - - hexadecimal digits are always uppercased (e.g. `0xBADC0DE`) - -- added `blackd`, see [its documentation](#blackd) for more info (#349) - -- adjacent string literals are now correctly split into multiple lines (#463) - -- trailing comma is now added to single imports that don't fit on a line (#250) - -- cache is now populated when `--check` is successful for a file which speeds up - consecutive checks of properly formatted unmodified files (#448) - -- whitespace at the beginning of the file is now removed (#399) - -- fixed mangling [pweave](http://mpastell.com/pweave/) and - [Spyder IDE](https://www.spyder-ide.org/) special comments (#532) - -- fixed unstable formatting when unpacking big tuples (#267) - -- fixed parsing of `__future__` imports with renames (#389) - -- fixed scope of `# fmt: off` when directly preceding `yield` and other nodes (#385) - -- fixed formatting of lambda expressions with default arguments (#468) - -- fixed `async for` statements: _Black_ no longer breaks them into separate lines (#372) - -- note: the Vim plugin stopped registering `,=` as a default chord as it turned out to - be a bad idea (#415) - -### 18.6b4 - -- hotfix: don't freeze when multiple comments directly precede `# fmt: off` (#371) - -### 18.6b3 - -- typing stub files (`.pyi`) now have blank lines added after constants (#340) - -- `# fmt: off` and `# fmt: on` are now much more dependable: - - - they now work also within bracket pairs (#329) - - - they now correctly work across function/class boundaries (#335) - - - they now work when an indentation block starts with empty lines or misaligned - comments (#334) - -- made Click not fail on invalid environments; note that Click is right but the - likelihood we'll need to access non-ASCII file paths when dealing with Python source - code is low (#277) - -- fixed improper formatting of f-strings with quotes inside interpolated expressions - (#322) - -- fixed unnecessary slowdown when long list literals where found in a file - -- fixed unnecessary slowdown on AST nodes with very many siblings - -- fixed cannibalizing backslashes during string normalization - -- fixed a crash due to symbolic links pointing outside of the project directory (#338) - -### 18.6b2 - -- added `--config` (#65) - -- added `-h` equivalent to `--help` (#316) - -- fixed improper unmodified file caching when `-S` was used - -- fixed extra space in string unpacking (#305) - -- fixed formatting of empty triple quoted strings (#313) - -- fixed unnecessary slowdown in comment placement calculation on lines without comments - -### 18.6b1 - -- hotfix: don't output human-facing information on stdout (#299) - -- hotfix: don't output cake emoji on non-zero return code (#300) - -### 18.6b0 - -- added `--include` and `--exclude` (#270) - -- added `--skip-string-normalization` (#118) - -- added `--verbose` (#283) - -- the header output in `--diff` now actually conforms to the unified diff spec - -- fixed long trivial assignments being wrapped in unnecessary parentheses (#273) - -- fixed unnecessary parentheses when a line contained multiline strings (#232) - -- fixed stdin handling not working correctly if an old version of Click was used (#276) - -- _Black_ now preserves line endings when formatting a file in place (#258) - -### 18.5b1 - -- added `--pyi` (#249) - -- added `--py36` (#249) - -- Python grammar pickle caches are stored with the formatting caches, making _Black_ - work in environments where site-packages is not user-writable (#192) - -- _Black_ now enforces a PEP 257 empty line after a class-level docstring (and/or - fields) and the first method - -- fixed invalid code produced when standalone comments were present in a trailer that - was omitted from line splitting on a large expression (#237) - -- fixed optional parentheses being removed within `# fmt: off` sections (#224) - -- fixed invalid code produced when stars in very long imports were incorrectly wrapped - in optional parentheses (#234) - -- fixed unstable formatting when inline comments were moved around in a trailer that was - omitted from line splitting on a large expression (#238) - -- fixed extra empty line between a class declaration and the first method if no class - docstring or fields are present (#219) - -- fixed extra empty line between a function signature and an inner function or inner - class (#196) - -### 18.5b0 - -- call chains are now formatted according to the - [fluent interfaces](https://en.wikipedia.org/wiki/Fluent_interface) style (#67) - -- data structure literals (tuples, lists, dictionaries, and sets) are now also always - exploded like imports when they don't fit in a single line (#152) - -- slices are now formatted according to PEP 8 (#178) - -- parentheses are now also managed automatically on the right-hand side of assignments - and return statements (#140) - -- math operators now use their respective priorities for delimiting multiline - expressions (#148) - -- optional parentheses are now omitted on expressions that start or end with a bracket - and only contain a single operator (#177) - -- empty parentheses in a class definition are now removed (#145, #180) - -- string prefixes are now standardized to lowercase and `u` is removed on Python 3.6+ - only code and Python 2.7+ code with the `unicode_literals` future import (#188, #198, - #199) - -- typing stub files (`.pyi`) are now formatted in a style that is consistent with PEP - 484 (#207, #210) - -- progress when reformatting many files is now reported incrementally - -- fixed trailers (content with brackets) being unnecessarily exploded into their own - lines after a dedented closing bracket (#119) - -- fixed an invalid trailing comma sometimes left in imports (#185) - -- fixed non-deterministic formatting when multiple pairs of removable parentheses were - used (#183) - -- fixed multiline strings being unnecessarily wrapped in optional parentheses in long - assignments (#215) - -- fixed not splitting long from-imports with only a single name - -- fixed Python 3.6+ file discovery by also looking at function calls with unpacking. - This fixed non-deterministic formatting if trailing commas where used both in function - signatures with stars and function calls with stars but the former would be - reformatted to a single line. - -- fixed crash on dealing with optional parentheses (#193) - -- fixed "is", "is not", "in", and "not in" not considered operators for splitting - purposes - -- fixed crash when dead symlinks where encountered - -### 18.4a4 - -- don't populate the cache on `--check` (#175) - -### 18.4a3 - -- added a "cache"; files already reformatted that haven't changed on disk won't be - reformatted again (#109) - -- `--check` and `--diff` are no longer mutually exclusive (#149) - -- generalized star expression handling, including double stars; this fixes - multiplication making expressions "unsafe" for trailing commas (#132) - -- _Black_ no longer enforces putting empty lines behind control flow statements (#90) - -- _Black_ now splits imports like "Mode 3 + trailing comma" of isort (#127) - -- fixed comment indentation when a standalone comment closes a block (#16, #32) - -- fixed standalone comments receiving extra empty lines if immediately preceding a - class, def, or decorator (#56, #154) - -- fixed `--diff` not showing entire path (#130) - -- fixed parsing of complex expressions after star and double stars in function calls - (#2) - -- fixed invalid splitting on comma in lambda arguments (#133) - -- fixed missing splits of ternary expressions (#141) - -### 18.4a2 - -- fixed parsing of unaligned standalone comments (#99, #112) - -- fixed placement of dictionary unpacking inside dictionary literals (#111) - -- Vim plugin now works on Windows, too - -- fixed unstable formatting when encountering unnecessarily escaped quotes in a string - (#120) - -### 18.4a1 - -- added `--quiet` (#78) - -- added automatic parentheses management (#4) - -- added [pre-commit](https://pre-commit.com) integration (#103, #104) - -- fixed reporting on `--check` with multiple files (#101, #102) - -- fixed removing backslash escapes from raw strings (#100, #105) - -### 18.4a0 - -- added `--diff` (#87) - -- add line breaks before all delimiters, except in cases like commas, to better comply - with PEP 8 (#73) - -- standardize string literals to use double quotes (almost) everywhere (#75) - -- fixed handling of standalone comments within nested bracketed expressions; _Black_ - will no longer produce super long lines or put all standalone comments at the end of - the expression (#22) - -- fixed 18.3a4 regression: don't crash and burn on empty lines with trailing whitespace - (#80) - -- fixed 18.3a4 regression: `# yapf: disable` usage as trailing comment would cause - _Black_ to not emit the rest of the file (#95) - -- when CTRL+C is pressed while formatting many files, _Black_ no longer freaks out with - a flurry of asyncio-related exceptions - -- only allow up to two empty lines on module level and only single empty lines within - functions (#74) - -### 18.3a4 - -- `# fmt: off` and `# fmt: on` are implemented (#5) - -- automatic detection of deprecated Python 2 forms of print statements and exec - statements in the formatted file (#49) - -- use proper spaces for complex expressions in default values of typed function - arguments (#60) - -- only return exit code 1 when --check is used (#50) - -- don't remove single trailing commas from square bracket indexing (#59) - -- don't omit whitespace if the previous factor leaf wasn't a math operator (#55) - -- omit extra space in kwarg unpacking if it's the first argument (#46) - -- omit extra space in - [Sphinx auto-attribute comments](http://www.sphinx-doc.org/en/stable/ext/autodoc.html#directive-autoattribute) - (#68) - -### 18.3a3 - -- don't remove single empty lines outside of bracketed expressions (#19) - -- added ability to pipe formatting from stdin to stdin (#25) - -- restored ability to format code with legacy usage of `async` as a name (#20, #42) - -- even better handling of numpy-style array indexing (#33, again) - -### 18.3a2 - -- changed positioning of binary operators to occur at beginning of lines instead of at - the end, following - [a recent change to PEP 8](https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b) - (#21) - -- ignore empty bracket pairs while splitting. This avoids very weirdly looking - formattings (#34, #35) - -- remove a trailing comma if there is a single argument to a call - -- if top level functions were separated by a comment, don't put four empty lines after - the upper function - -- fixed unstable formatting of newlines with imports - -- fixed unintentional folding of post scriptum standalone comments into last statement - if it was a simple statement (#18, #28) - -- fixed missing space in numpy-style array indexing (#33) - -- fixed spurious space after star-based unary expressions (#31) - -### 18.3a1 - -- added `--check` - -- only put trailing commas in function signatures and calls if it's safe to do so. If - the file is Python 3.6+ it's always safe, otherwise only safe if there are no `*args` - or `**kwargs` used in the signature or call. (#8) - -- fixed invalid spacing of dots in relative imports (#6, #13) - -- fixed invalid splitting after comma on unpacked variables in for-loops (#23) - -- fixed spurious space in parenthesized set expressions (#7) - -- fixed spurious space after opening parentheses and in default arguments (#14, #17) - -- fixed spurious space after unary operators when the operand was a complex expression - (#15) - -### 18.3a0 - -- first published version, Happy 🍰 Day 2018! - -- alpha quality - -- date-versioned (see: https://calver.org/) diff --git a/docs/change_log.md b/docs/change_log.md new file mode 120000 index 00000000000..cf547089dc1 --- /dev/null +++ b/docs/change_log.md @@ -0,0 +1 @@ +../CHANGES.md \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 7381c9d6423..ec6aad9a27b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -219,7 +219,6 @@ def process_sections( DocSection("blackd", CURRENT_DIR / "blackd.md"), DocSection("black_primer", CURRENT_DIR / "black_primer.md"), DocSection("contributing_to_black", CURRENT_DIR / ".." / "CONTRIBUTING.md"), - DocSection("change_log", CURRENT_DIR / ".." / "CHANGES.md"), ] # Sphinx complains when there is a source file that isn't referenced in any of the docs. @@ -231,6 +230,7 @@ def process_sections( "pragmatism", "testimonials", "used_by", + "change_log", } make_pypi_svg(release) From 557b54aa60e9abbb93e5737512d9963e640f2d1f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 26 Apr 2021 10:42:16 -0700 Subject: [PATCH 26/81] Fix crash on docstrings ending with "\ " (#2142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- CHANGES.md | 2 ++ src/black/__init__.py | 8 +++++++- tests/data/docstring.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9fe6b7e3fa4..814956c80ba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ #### _Black_ +- Fix crash on docstrings ending with "\ ". (#2142) + - Reflect the `--skip-magic-trailing-comma` and `--experimental-string-processing` flags in the name of the cache file. Without this fix, changes in these flags would not take effect if the cache had already been populated. (#2131) diff --git a/src/black/__init__.py b/src/black/__init__.py index 99afc7d3c50..051de5d13df 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -2186,7 +2186,13 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if docstring[0] == quote_char: docstring = " " + docstring if docstring[-1] == quote_char: - docstring = docstring + " " + docstring += " " + if docstring[-1] == "\\": + backslash_count = len(docstring) - len(docstring.rstrip("\\")) + if backslash_count % 2: + # Odd number of tailing backslashes, add some padding to + # avoid escaping the closing string quote. + docstring += " " else: # Add some padding if the docstring is empty. docstring = " " diff --git a/tests/data/docstring.py b/tests/data/docstring.py index 9e1c2441e0e..ee6d0c051d7 100644 --- a/tests/data/docstring.py +++ b/tests/data/docstring.py @@ -158,6 +158,22 @@ def docstring_with_inline_tabs_and_tab_indentation(): """ pass + +def backslash_space(): + """\ """ + + +def multiline_backslash_1(): + ''' + hey\there\ + \ ''' + + +def multiline_backslash_2(): + ''' + hey there \ ''' + + # output class MyClass: @@ -316,3 +332,18 @@ def docstring_with_inline_tabs_and_tab_indentation(): line ends with some tabs """ pass + + +def backslash_space(): + """\ """ + + +def multiline_backslash_1(): + """ + hey\there\ + \ """ + + +def multiline_backslash_2(): + """ + hey there \ """ From eaa337f176b086a9ebd91884c0b9d9a96772aeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Apr 2021 20:24:06 +0200 Subject: [PATCH 27/81] Add more tests for fancy whitespace (#2147) --- CHANGES.md | 2 + tests/data/docstring.py | 28 +++++++++++++ .../data/docstring_no_string_normalization.py | 40 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 814956c80ba..a3bf5565967 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ - Fix crash on docstrings ending with "\ ". (#2142) +- Fix crash when atypical whitespace is cleaned out of dostrings (#2120) + - Reflect the `--skip-magic-trailing-comma` and `--experimental-string-processing` flags in the name of the cache file. Without this fix, changes in these flags would not take effect if the cache had already been populated. (#2131) diff --git a/tests/data/docstring.py b/tests/data/docstring.py index ee6d0c051d7..e977619e482 100644 --- a/tests/data/docstring.py +++ b/tests/data/docstring.py @@ -174,6 +174,20 @@ def multiline_backslash_2(): hey there \ ''' +def multiline_backslash_3(): + ''' + already escaped \\ ''' + + +def my_god_its_full_of_stars_1(): + "I'm sorry Dave\u2001" + + +# the space below is actually a \u2001, removed in output +def my_god_its_full_of_stars_2(): + "I'm sorry Dave " + + # output class MyClass: @@ -347,3 +361,17 @@ def multiline_backslash_1(): def multiline_backslash_2(): """ hey there \ """ + + +def multiline_backslash_3(): + """ + already escaped \\""" + + +def my_god_its_full_of_stars_1(): + "I'm sorry Dave\u2001" + + +# the space below is actually a \u2001, removed in output +def my_god_its_full_of_stars_2(): + "I'm sorry Dave" \ No newline at end of file diff --git a/tests/data/docstring_no_string_normalization.py b/tests/data/docstring_no_string_normalization.py index 0457fcf114f..a90b578f09a 100644 --- a/tests/data/docstring_no_string_normalization.py +++ b/tests/data/docstring_no_string_normalization.py @@ -102,6 +102,26 @@ def shockingly_the_quotes_are_normalized_v2(): ''' pass + +def backslash_space(): + '\ ' + + +def multiline_backslash_1(): + ''' + hey\there\ + \ ''' + + +def multiline_backslash_2(): + ''' + hey there \ ''' + + +def multiline_backslash_3(): + ''' + already escaped \\ ''' + # output class ALonelyClass: @@ -207,3 +227,23 @@ def shockingly_the_quotes_are_normalized_v2(): Docstring Docstring Docstring ''' pass + + +def backslash_space(): + '\ ' + + +def multiline_backslash_1(): + ''' + hey\there\ + \ ''' + + +def multiline_backslash_2(): + ''' + hey there \ ''' + + +def multiline_backslash_3(): + ''' + already escaped \\''' From a0b4e609a7393fac11ecc94eb859c1e85b540036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 26 Apr 2021 21:22:25 +0200 Subject: [PATCH 28/81] Bump pathspec to >= 0.8.1 to solve invalid .gitignore exclusion handling (#2084) Also made the Click requirement in Pipfile consistent with setup.py and bumped mypy. --- Pipfile | 6 +- Pipfile.lock | 697 +++++++++++++++++++++++++-------------------------- setup.py | 2 +- 3 files changed, 344 insertions(+), 361 deletions(-) diff --git a/Pipfile b/Pipfile index 32fb867415f..40f3d607653 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ coverage = "*" docutils = "==0.15" # not a direct dependency, see https://github.com/pypa/pipenv/issues/3865 flake8 = "*" flake8-bugbear = "*" -mypy = ">=0.782" +mypy = ">=0.812" pre-commit = "*" readme_renderer = "*" recommonmark = "*" @@ -23,9 +23,9 @@ black = {editable = true, extras = ["d"], path = "."} aiohttp = ">=3.3.2" aiohttp-cors = "*" appdirs = "*" -click = ">=7.0" +click = ">=7.1.2" mypy_extensions = ">=0.4.3" -pathspec = ">=0.6" +pathspec = ">=0.8.1" regex = ">=2020.1.8" toml = ">=0.10.1" typed-ast = "==1.4.2" diff --git a/Pipfile.lock b/Pipfile.lock index 3d908dc90f3..8a16a75026a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -16,46 +16,46 @@ "default": { "aiohttp": { "hashes": [ - "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0", - "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6", - "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf", - "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9", - "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e", - "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0", - "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329", - "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2", - "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40", - "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a", - "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4", - "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de", - "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9", - "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9", - "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb", - "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076", - "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de", - "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907", - "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d", - "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536", - "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d", - "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54", - "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc", - "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212", - "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9", - "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d", - "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b", - "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7", - "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81", - "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c", - "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895", - "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297", - "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb", - "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe", - "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242", - "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0", - "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2" + "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe", + "sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe", + "sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5", + "sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8", + "sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd", + "sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb", + "sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c", + "sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87", + "sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0", + "sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290", + "sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5", + "sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287", + "sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde", + "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf", + "sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8", + "sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16", + "sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf", + "sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809", + "sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213", + "sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f", + "sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013", + "sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b", + "sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9", + "sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5", + "sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb", + "sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df", + "sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4", + "sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439", + "sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f", + "sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22", + "sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f", + "sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5", + "sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970", + "sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009", + "sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc", + "sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a", + "sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95" ], "index": "pypi", - "version": "==3.7.4" + "version": "==3.7.4.post0" }, "aiohttp-cors": { "hashes": [ @@ -78,6 +78,7 @@ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], + "markers": "python_full_version >= '3.5.3'", "version": "==3.0.1" }, "attrs": { @@ -85,6 +86,7 @@ "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.3.0" }, "black": { @@ -96,10 +98,11 @@ }, "chardet": { "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" ], - "version": "==3.0.4" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" }, "click": { "hashes": [ @@ -115,13 +118,16 @@ "sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84" ], "index": "pypi", - "version": "==0.6" + "python_version <": "3.7", + "version": "==0.6", + "version >": "0.6" }, "idna": { "hashes": [ "sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16", "sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1" ], + "markers": "python_version >= '3.4'", "version": "==3.1" }, "multidict": { @@ -164,6 +170,7 @@ "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281", "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80" ], + "markers": "python_version >= '3.6'", "version": "==5.1.0" }, "mypy-extensions": { @@ -184,50 +191,58 @@ }, "regex": { "hashes": [ - "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", - "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", - "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", - "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", - "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", - "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", - "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", - "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", - "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", - "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", - "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", - "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", - "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", - "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", - "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", - "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", - "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", - "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", - "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", - "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", - "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", - "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", - "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", - "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", - "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", - "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", - "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", - "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", - "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", - "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", - "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", - "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", - "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", - "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", - "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", - "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", - "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", - "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", - "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", - "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", - "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", + "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", + "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", + "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", + "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", + "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", + "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", + "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", + "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", + "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", + "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", + "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", + "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", + "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", + "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", + "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", + "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", + "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", + "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", + "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", + "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", + "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", + "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", + "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", + "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", + "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", + "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", + "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", + "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", + "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", + "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", + "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", + "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", + "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", + "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", + "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", + "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", + "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", + "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", + "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", + "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" ], "index": "pypi", - "version": "==2020.11.13" + "version": "==2021.4.4" + }, + "setuptools-scm": { + "hashes": [ + "sha256:c3bd5f701c8def44a5c0bfe8d407bef3f80342217ef3492b951f3777bd2d915c", + "sha256:d1925a69cb07e9b29416a275b9fadb009a23c148ace905b2fb220649a6c18e92" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0.1" }, "toml": { "hashes": [ @@ -280,7 +295,9 @@ "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" ], "index": "pypi", - "version": "==3.7.4.3" + "python_version <": "3.8", + "version": "==3.7.4.3", + "version >=": "3.7.4" }, "yarl": { "hashes": [ @@ -322,52 +339,53 @@ "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" ], + "markers": "python_version >= '3.6'", "version": "==1.6.3" } }, "develop": { "aiohttp": { "hashes": [ - "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0", - "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6", - "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf", - "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9", - "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e", - "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0", - "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329", - "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2", - "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40", - "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a", - "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4", - "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de", - "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9", - "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9", - "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb", - "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076", - "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de", - "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907", - "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d", - "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536", - "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d", - "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54", - "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc", - "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212", - "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9", - "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d", - "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b", - "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7", - "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81", - "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c", - "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895", - "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297", - "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb", - "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe", - "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242", - "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0", - "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2" + "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe", + "sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe", + "sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5", + "sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8", + "sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd", + "sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb", + "sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c", + "sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87", + "sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0", + "sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290", + "sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5", + "sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287", + "sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde", + "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf", + "sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8", + "sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16", + "sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf", + "sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809", + "sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213", + "sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f", + "sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013", + "sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b", + "sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9", + "sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5", + "sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb", + "sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df", + "sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4", + "sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439", + "sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f", + "sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22", + "sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f", + "sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5", + "sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970", + "sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009", + "sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc", + "sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a", + "sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95" ], "index": "pypi", - "version": "==3.7.4" + "version": "==3.7.4.post0" }, "aiohttp-cors": { "hashes": [ @@ -397,6 +415,7 @@ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], + "markers": "python_full_version >= '3.5.3'", "version": "==3.0.1" }, "attrs": { @@ -404,6 +423,7 @@ "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.3.0" }, "babel": { @@ -411,6 +431,7 @@ "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5", "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0" }, "black": { @@ -425,6 +446,7 @@ "sha256:6123ddc1052673e52bab52cdc955bcb57a015264a1c57d37bea2f6b817af0125", "sha256:98b3170739e5e83dd9dc19633f074727ad848cbedb6026708c8ac2d3b697a433" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.3.0" }, "certifi": { @@ -434,61 +456,21 @@ ], "version": "==2020.12.5" }, - "cffi": { - "hashes": [ - "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813", - "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06", - "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea", - "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee", - "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396", - "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73", - "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315", - "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1", - "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49", - "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892", - "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482", - "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058", - "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5", - "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53", - "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045", - "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3", - "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5", - "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e", - "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c", - "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369", - "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827", - "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053", - "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa", - "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4", - "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322", - "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132", - "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62", - "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa", - "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0", - "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396", - "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e", - "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991", - "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6", - "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1", - "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406", - "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d", - "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c" - ], - "version": "==1.14.5" - }, "cfgv": { "hashes": [ "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1" ], + "markers": "python_full_version >= '3.6.1'", "version": "==3.2.0" }, "chardet": { "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" ], - "version": "==3.0.4" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" }, "click": { "hashes": [ @@ -503,6 +485,7 @@ "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.4.4" }, "commonmark": { @@ -514,75 +497,61 @@ }, "coverage": { "hashes": [ - "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297", - "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1", - "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497", - "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606", - "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528", - "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b", - "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4", - "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830", - "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1", - "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f", - "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d", - "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3", - "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8", - "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500", - "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7", - "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb", - "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b", - "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059", - "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b", - "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72", - "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36", - "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277", - "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c", - "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631", - "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff", - "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8", - "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec", - "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b", - "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7", - "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105", - "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b", - "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c", - "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b", - "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98", - "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4", - "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879", - "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f", - "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4", - "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044", - "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e", - "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899", - "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f", - "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448", - "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714", - "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2", - "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d", - "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd", - "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7", - "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae" + "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", + "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6", + "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45", + "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a", + "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03", + "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529", + "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a", + "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a", + "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2", + "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6", + "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759", + "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53", + "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a", + "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4", + "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff", + "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502", + "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793", + "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb", + "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905", + "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821", + "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b", + "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81", + "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0", + "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b", + "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3", + "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184", + "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701", + "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a", + "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82", + "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638", + "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5", + "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083", + "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6", + "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90", + "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465", + "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a", + "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3", + "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e", + "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066", + "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf", + "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b", + "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae", + "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669", + "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873", + "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b", + "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6", + "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb", + "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160", + "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c", + "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079", + "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", + "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" ], "index": "pypi", - "version": "==5.3.1" - }, - "cryptography": { - "hashes": [ - "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", - "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", - "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", - "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", - "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", - "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", - "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", - "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", - "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", - "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", - "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", - "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" - ], - "version": "==3.4.7" + "version": "==5.5" }, "distlib": { "hashes": [ @@ -594,6 +563,7 @@ "docutils": { "hashes": [ "sha256:54a349c622ff31c91cbec43b0b512f113b5b24daf00e2ea530bb1bd9aac14849", + "sha256:ba4584f9107571ced0d2c7f56a5499c696215ba90797849c92d395979da68521", "sha256:d2ddba74835cb090a1b627d3de4e7835c628d07ee461f7b4480f51af2fe4d448" ], "index": "pypi", @@ -608,32 +578,34 @@ }, "flake8": { "hashes": [ - "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", - "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" + "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378", + "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a" ], "index": "pypi", - "version": "==3.8.4" + "version": "==3.9.1" }, "flake8-bugbear": { "hashes": [ - "sha256:528020129fea2dea33a466b9d64ab650aa3e5f9ffc788b70ea4bc6cf18283538", - "sha256:f35b8135ece7a014bc0aee5b5d485334ac30a6da48494998cc1fabf7ec70d703" + "sha256:2346c81f889955b39e4a368eb7d508de723d9de05716c287dc860a4073dc57e7", + "sha256:4f305dca96be62bf732a218fe6f1825472a621d3452c5b994d8f89dae21dbafa" ], "index": "pypi", - "version": "==20.11.1" + "version": "==21.4.3" }, "identify": { "hashes": [ - "sha256:43cb1965e84cdd247e875dec6d13332ef5be355ddc16776396d98089b9053d87", - "sha256:c7c0f590526008911ccc5ceee6ed7b085cbc92f7b6591d0ee5913a130ad64034" + "sha256:9bcc312d4e2fa96c7abebcdfb1119563b511b5e3985ac52f60d9116277865b2e", + "sha256:ad9f3fa0c2316618dc4d840f627d474ab6de106392a4f00221820200f490f5a8" ], - "version": "==2.2.2" + "markers": "python_full_version >= '3.6.1'", + "version": "==2.2.4" }, "idna": { "hashes": [ "sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16", "sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1" ], + "markers": "python_version >= '3.4'", "version": "==3.1" }, "imagesize": { @@ -641,28 +613,23 @@ "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1", "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.0" }, "importlib-metadata": { "hashes": [ - "sha256:c9db46394197244adf2f0b08ec5bc3cf16757e9590b02af1fca085c16c0d600a", - "sha256:d2d46ef77ffc85cbf7dac7e81dd663fde71c45326131bea8033b9bad42268ebe" - ], - "version": "==3.10.0" - }, - "jeepney": { - "hashes": [ - "sha256:7d59b6622675ca9e993a6bd38de845051d315f8b0c72cca3aef733a20b648657", - "sha256:aec56c0eb1691a841795111e184e13cad504f7703b9a64f63020816afa79a8ae" + "sha256:8c501196e49fb9df5df43833bdb1e4328f64847763ec8a50703148b73784d581", + "sha256:d7eb1dea6d6a6086f8be21784cc9e3bcfa55872b52309bc5fad53a8ea444465d" ], - "markers": "sys_platform == 'linux'", - "version": "==0.6.0" + "markers": "python_version >= '3.6'", + "version": "==4.0.1" }, "jinja2": { "hashes": [ "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.11.3" }, "keyring": { @@ -670,6 +637,7 @@ "sha256:045703609dd3fccfcdb27da201684278823b72af515aedec1a8515719a038cb8", "sha256:8f607d7d1cc502c43a932a275a56fe47db50271904513a379d39df1af277ac48" ], + "markers": "python_version >= '3.6'", "version": "==23.0.1" }, "markupsafe": { @@ -727,6 +695,7 @@ "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "mccabe": { @@ -776,27 +745,36 @@ "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281", "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80" ], + "markers": "python_version >= '3.6'", "version": "==5.1.0" }, "mypy": { "hashes": [ - "sha256:0a0d102247c16ce93c97066443d11e2d36e6cc2a32d8ccc1f705268970479324", - "sha256:0d34d6b122597d48a36d6c59e35341f410d4abfa771d96d04ae2c468dd201abc", - "sha256:2170492030f6faa537647d29945786d297e4862765f0b4ac5930ff62e300d802", - "sha256:2842d4fbd1b12ab422346376aad03ff5d0805b706102e475e962370f874a5122", - "sha256:2b21ba45ad9ef2e2eb88ce4aeadd0112d0f5026418324176fd494a6824b74975", - "sha256:72060bf64f290fb629bd4a67c707a66fd88ca26e413a91384b18db3876e57ed7", - "sha256:af4e9ff1834e565f1baa74ccf7ae2564ae38c8df2a85b057af1dbbc958eb6666", - "sha256:bd03b3cf666bff8d710d633d1c56ab7facbdc204d567715cb3b9f85c6e94f669", - "sha256:c614194e01c85bb2e551c421397e49afb2872c88b5830e3554f0519f9fb1c178", - "sha256:cf4e7bf7f1214826cf7333627cb2547c0db7e3078723227820d0a2490f117a01", - "sha256:da56dedcd7cd502ccd3c5dddc656cb36113dd793ad466e894574125945653cea", - "sha256:e86bdace26c5fe9cf8cb735e7cedfe7850ad92b327ac5d797c656717d2ca66de", - "sha256:e97e9c13d67fbe524be17e4d8025d51a7dca38f90de2e462243ab8ed8a9178d1", - "sha256:eea260feb1830a627fb526d22fbb426b750d9f5a47b624e8d5e7e004359b219c" + "sha256:0d0a87c0e7e3a9becdfbe936c981d32e5ee0ccda3e0f07e1ef2c3d1a817cf73e", + "sha256:25adde9b862f8f9aac9d2d11971f226bd4c8fbaa89fb76bdadb267ef22d10064", + "sha256:28fb5479c494b1bab244620685e2eb3c3f988d71fd5d64cc753195e8ed53df7c", + "sha256:2f9b3407c58347a452fc0736861593e105139b905cca7d097e413453a1d650b4", + "sha256:33f159443db0829d16f0a8d83d94df3109bb6dd801975fe86bacb9bf71628e97", + "sha256:3f2aca7f68580dc2508289c729bd49ee929a436208d2b2b6aab15745a70a57df", + "sha256:499c798053cdebcaa916eef8cd733e5584b5909f789de856b482cd7d069bdad8", + "sha256:4eec37370483331d13514c3f55f446fc5248d6373e7029a29ecb7b7494851e7a", + "sha256:552a815579aa1e995f39fd05dde6cd378e191b063f031f2acfe73ce9fb7f9e56", + "sha256:5873888fff1c7cf5b71efbe80e0e73153fe9212fafdf8e44adfe4c20ec9f82d7", + "sha256:61a3d5b97955422964be6b3baf05ff2ce7f26f52c85dd88db11d5e03e146a3a6", + "sha256:674e822aa665b9fd75130c6c5f5ed9564a38c6cea6a6432ce47eafb68ee578c5", + "sha256:7ce3175801d0ae5fdfa79b4f0cfed08807af4d075b402b7e294e6aa72af9aa2a", + "sha256:9743c91088d396c1a5a3c9978354b61b0382b4e3c440ce83cf77994a43e8c521", + "sha256:9f94aac67a2045ec719ffe6111df543bac7874cee01f41928f6969756e030564", + "sha256:a26f8ec704e5a7423c8824d425086705e381b4f1dfdef6e3a1edab7ba174ec49", + "sha256:abf7e0c3cf117c44d9285cc6128856106183938c68fd4944763003decdcfeb66", + "sha256:b09669bcda124e83708f34a94606e01b614fa71931d356c1f1a5297ba11f110a", + "sha256:cd07039aa5df222037005b08fbbfd69b3ab0b0bd7a07d7906de75ae52c4e3119", + "sha256:d23e0ea196702d918b60c8288561e722bf437d82cb7ef2edcd98cfa38905d506", + "sha256:d65cc1df038ef55a99e617431f0553cd77763869eebdf9042403e16089fe746c", + "sha256:d7da2e1d5f558c37d6e8c1246f1aec1e7349e4913d8fb3cb289a35de573fe2eb" ], "index": "pypi", - "version": "==0.790" + "version": "==0.812" }, "mypy-extensions": { "hashes": [ @@ -808,16 +786,17 @@ }, "nodeenv": { "hashes": [ - "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9", - "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c" + "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b", + "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7" ], - "version": "==1.5.0" + "version": "==1.6.0" }, "packaging": { "hashes": [ "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.9" }, "pathspec": { @@ -837,38 +816,34 @@ }, "pre-commit": { "hashes": [ - "sha256:6c86d977d00ddc8a60d68eec19f51ef212d9462937acf3ea37c7adec32284ac0", - "sha256:ee784c11953e6d8badb97d19bc46b997a3a9eded849881ec587accd8608d74a4" + "sha256:70c5ec1f30406250b706eda35e868b87e3e4ba099af8787e3e8b4b01e84f4712", + "sha256:900d3c7e1bf4cf0374bb2893c24c23304952181405b4d88c9c40b72bda1bb8a9" ], "index": "pypi", - "version": "==2.9.3" + "version": "==2.12.1" }, "pycodestyle": { "hashes": [ - "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", - "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068", + "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef" ], - "version": "==2.6.0" - }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "version": "==2.20" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.7.0" }, "pyflakes": { "hashes": [ - "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", - "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", + "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db" ], - "version": "==2.2.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.3.1" }, "pygments": { "hashes": [ "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94", "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8" ], + "markers": "python_version >= '3.5'", "version": "==2.8.1" }, "pyparsing": { @@ -876,6 +851,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytz": { @@ -917,15 +893,16 @@ "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==5.4.1" }, "readme-renderer": { "hashes": [ - "sha256:267854ac3b1530633c2394ead828afcd060fc273217c42ac36b6be9c42cd9a9d", - "sha256:6b7e5aa59210a40de72eb79931491eaf46fefca2952b9181268bd7c7c65c260a" + "sha256:63b4075c6698fcfa78e584930f07f39e05d46f3ec97f65006e430b595ca6348c", + "sha256:92fd5ac2bf8677f310f3303aa4bce5b9d5f9f2094ab98c29f13791d7b805a3db" ], "index": "pypi", - "version": "==28.0" + "version": "==29.0" }, "recommonmark": { "hashes": [ @@ -937,56 +914,57 @@ }, "regex": { "hashes": [ - "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", - "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", - "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", - "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", - "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", - "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", - "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", - "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", - "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", - "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", - "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", - "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", - "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", - "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", - "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", - "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", - "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", - "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", - "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", - "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", - "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", - "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", - "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", - "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", - "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", - "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", - "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", - "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", - "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", - "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", - "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", - "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", - "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", - "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", - "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", - "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", - "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", - "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", - "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", - "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", - "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", + "sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", + "sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", + "sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", + "sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", + "sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", + "sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", + "sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", + "sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", + "sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", + "sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", + "sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", + "sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", + "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", + "sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", + "sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", + "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", + "sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", + "sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", + "sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", + "sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", + "sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", + "sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", + "sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", + "sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", + "sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", + "sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", + "sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", + "sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", + "sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", + "sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", + "sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", + "sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", + "sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", + "sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", + "sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", + "sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", + "sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", + "sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", + "sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", + "sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" ], "index": "pypi", - "version": "==2020.11.13" + "version": "==2021.4.4" }, "requests": { "hashes": [ "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.25.1" }, "requests-toolbelt": { @@ -1003,27 +981,20 @@ ], "version": "==1.4.0" }, - "secretstorage": { - "hashes": [ - "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", - "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" - ], - "markers": "sys_platform == 'linux'", - "version": "==3.3.1" - }, "setuptools-scm": { "hashes": [ - "sha256:62fa535edb31ece9fa65dc9dcb3056145b8020c8c26c0ef1018aef33db95c40d", - "sha256:c85b6b46d0edd40d2301038cdea96bb6adc14d62ef943e75afb08b3e7bcf142a" + "sha256:c3bd5f701c8def44a5c0bfe8d407bef3f80342217ef3492b951f3777bd2d915c", + "sha256:d1925a69cb07e9b29416a275b9fadb009a23c148ace905b2fb220649a6c18e92" ], - "index": "pypi", - "version": "==5.0.1" + "markers": "python_version >= '3.6'", + "version": "==6.0.1" }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "snowballstemmer": { @@ -1035,17 +1006,18 @@ }, "sphinx": { "hashes": [ - "sha256:aeef652b14629431c82d3fe994ce39ead65b3fe87cf41b9a3714168ff8b83376", - "sha256:e450cb205ff8924611085183bf1353da26802ae73d9251a8fcdf220a8f8712ef" + "sha256:19010b7b9fa0dc7756a6e105b2aacd3a80f798af3c25c273be64d7beeb482cb1", + "sha256:2320d4e994a191f4b4be27da514e46b3d6b420f2ff895d064f52415d342461e8" ], "index": "pypi", - "version": "==3.4.1" + "version": "==3.5.4" }, "sphinxcontrib-applehelp": { "hashes": [ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-devhelp": { @@ -1053,6 +1025,7 @@ "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-htmlhelp": { @@ -1060,6 +1033,7 @@ "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f", "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b" ], + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-jsmath": { @@ -1067,6 +1041,7 @@ "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" ], + "markers": "python_version >= '3.5'", "version": "==1.0.1" }, "sphinxcontrib-qthelp": { @@ -1074,6 +1049,7 @@ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" ], + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-serializinghtml": { @@ -1081,6 +1057,7 @@ "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc", "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a" ], + "markers": "python_version >= '3.5'", "version": "==1.1.4" }, "toml": { @@ -1096,15 +1073,16 @@ "sha256:daec693491c52e9498632dfbe9ccfc4882a557f5fa08982db1b4d3adbe0887c3", "sha256:ebdebdb95e3477ceea267decfc0784859aa3df3e27e22d23b83e9b272bf157ae" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==4.60.0" }, "twine": { "hashes": [ - "sha256:2f6942ec2a17417e19d2dd372fc4faa424c87ee9ce49b4e20c427eb00a0f3f41", - "sha256:fcffa8fc37e8083a5be0728371f299598870ee1eccc94e9a25cef7b1dcfa8297" + "sha256:16f706f2f1687d7ce30e7effceee40ed0a09b7c33b9abb5ef6434e5551565d83", + "sha256:a56c985264b991dc8a8f4234eb80c5af87fa8080d0c224ad8f2cd05a2c22e83b" ], "index": "pypi", - "version": "==3.3.0" + "version": "==3.4.1" }, "typed-ast": { "hashes": [ @@ -1149,22 +1127,25 @@ "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" ], "index": "pypi", - "version": "==3.7.4.3" + "python_version <": "3.8", + "version": "==3.7.4.3", + "version >=": "3.7.4" }, "urllib3": { "hashes": [ "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], - "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" }, "virtualenv": { "hashes": [ - "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107", - "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c" + "sha256:09c61377ef072f43568207dc8e46ddeac6bcdcaf288d49011bda0e7f4d38c4a2", + "sha256:a935126db63128861987a7d5d30e23e8ec045a73840eeccb467c148514e29535" ], - "version": "==20.4.3" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.4.4" }, "webencodings": { "hashes": [ @@ -1221,6 +1202,7 @@ "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" ], + "markers": "python_version >= '3.6'", "version": "==1.6.3" }, "zipp": { @@ -1228,6 +1210,7 @@ "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" ], + "markers": "python_version >= '3.6'", "version": "==3.4.1" } } diff --git a/setup.py b/setup.py index 856c7fadb0c..0928c63afc2 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_long_description() -> str: "toml>=0.10.1", "typed-ast>=1.4.2; python_version < '3.8'", "regex>=2020.1.8", - "pathspec>=0.6, <1", + "pathspec>=0.8.1, <1", "dataclasses>=0.6; python_version < '3.7'", "typing_extensions>=3.7.4; python_version < '3.8'", "mypy_extensions>=0.4.3", From b55ea63ff4858109fe4606a6caafb03fda946743 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:26:43 -0400 Subject: [PATCH 29/81] Stop stripping parens in even more illegal spots (#2148) We're only fixing them so fuzzers don't yell at us when we break "valid" code. I mean "valid" because some of the examples aren't even accepted by Python. --- src/black/__init__.py | 3 +++ tests/data/pep_572_do_not_remove_parens.py | 21 +++++++++++++++++++++ tests/data/pep_572_remove_parens.py | 15 +++++++++++++++ tests/test_black.py | 8 ++++++++ 4 files changed, 47 insertions(+) create mode 100644 tests/data/pep_572_do_not_remove_parens.py diff --git a/src/black/__init__.py b/src/black/__init__.py index 051de5d13df..015ca415f1c 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5624,6 +5624,9 @@ def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: syms.expr_stmt, syms.assert_stmt, syms.return_stmt, + # these ones aren't useful to end users, but they do please fuzzers + syms.for_stmt, + syms.del_stmt, ]: return False diff --git a/tests/data/pep_572_do_not_remove_parens.py b/tests/data/pep_572_do_not_remove_parens.py new file mode 100644 index 00000000000..20e80a69377 --- /dev/null +++ b/tests/data/pep_572_do_not_remove_parens.py @@ -0,0 +1,21 @@ +# Most of the following examples are really dumb, some of them aren't even accepted by Python, +# we're fixing them only so fuzzers (which follow the grammar which actually allows these +# examples matter of fact!) don't yell at us :p + +del (a := [1]) + +try: + pass +except (a := 1) as (b := why_does_this_exist): + pass + +for (z := 124) in (x := -124): + pass + +with (y := [3, 2, 1]) as (funfunfun := indeed): + pass + + +@(please := stop) +def sigh(): + pass diff --git a/tests/data/pep_572_remove_parens.py b/tests/data/pep_572_remove_parens.py index 9a804859a09..9718d95b499 100644 --- a/tests/data/pep_572_remove_parens.py +++ b/tests/data/pep_572_remove_parens.py @@ -33,6 +33,9 @@ def foo2(answer: (p := 42) = 5): lambda: (x := 1) +a[(x := 12)] +a[:(x := 13)] + # we don't touch expressions in f-strings but if we do one day, don't break 'em f'{(x:=10)}' @@ -43,6 +46,10 @@ def a(): yield (a := 2) raise (c := 3) +def this_is_so_dumb() -> (please := no): + pass + + # output if foo := 0: pass @@ -79,6 +86,9 @@ def foo2(answer: (p := 42) = 5): lambda: (x := 1) +a[(x := 12)] +a[:(x := 13)] + # we don't touch expressions in f-strings but if we do one day, don't break 'em f"{(x:=10)}" @@ -88,3 +98,8 @@ def a(): await (b := 1) yield (a := 2) raise (c := 3) + + +def this_is_so_dumb() -> (please := no): + pass + diff --git a/tests/test_black.py b/tests/test_black.py index 88ea31777fb..c643f27cacf 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -290,6 +290,14 @@ def test_pep_572_remove_parens(self) -> None: if sys.version_info >= (3, 8): black.assert_equivalent(source, actual) + @patch("black.dump_to_file", dump_to_stderr) + def test_pep_572_do_not_remove_parens(self) -> None: + source, expected = read_data("pep_572_do_not_remove_parens") + # the AST safety checks will fail, but that's expected, just make sure no + # parentheses are touched + actual = black.format_str(source, mode=DEFAULT_MODE) + self.assertFormatEqual(expected, actual) + def test_pep_572_version_detection(self) -> None: source, _ = read_data("pep_572") root = black.lib2to3_parse(source) From 303a0b2c4d0c828ba6cd4ef6d480f1ff2b5a11c6 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:59:45 -0400 Subject: [PATCH 30/81] Maintainers += Richard Si (aka ichard26) (#2149) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ed9f105f5aa..8d6adb0567e 100644 --- a/README.md +++ b/README.md @@ -539,8 +539,8 @@ Maintained with [Carol Willing](mailto:carolcode@willingconsulting.com), [Carl Meyer](mailto:carl@oddbird.net), [Jelle Zijlstra](mailto:jelle.zijlstra@gmail.com), [Mika Naylor](mailto:mail@autophagy.io), -[Zsolt Dollenstein](mailto:zsol.zsol@gmail.com), and -[Cooper Lees](mailto:me@cooperlees.com). +[Zsolt Dollenstein](mailto:zsol.zsol@gmail.com), +[Cooper Lees](mailto:me@cooperlees.com), and Richard Si. Multiple contributions by: From 1728bb441e6cc3ba5a5ce52ec62014bbfa435a82 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 27 Apr 2021 07:40:08 -0700 Subject: [PATCH 31/81] Update CHANGELOG for 21.4b1 release (#2151) * Update CHANGELOG for 21.4b1 release * Add pathspec minimum bump + update primer not to expect changes for virtualenv --- CHANGES.md | 6 +++++- src/black_primer/primer.json | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3bf5565967..d114c5136ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ ## Change Log -### Unreleased +### 21.4b1 #### _Black_ @@ -15,6 +15,10 @@ - Don't remove necessary parentheses from assignment expression containing assert / return statements. (#2143) +#### _Packaging_ + +- Bump pathspec to >= 0.8.1 to solve invalid .gitignore exclusion handling + ### 21.4b0 #### _Black_ diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 7a6f5abf408..d38e01dd707 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -112,7 +112,7 @@ }, "virtualenv": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/pypa/virtualenv.git", "long_checkout": false, "py_versions": ["all"] From 82a53999ea971895e2836df39be1ec9110ec102c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 27 Apr 2021 08:33:51 -0700 Subject: [PATCH 32/81] Add pyanalyze and typeshed to black-primer (#2152) pyanalyze is one of my projects and it uses `--experimental-string-processing`. typeshed has a lot of stub files. --- src/black_primer/primer.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index d38e01dd707..75469ad815b 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -75,6 +75,13 @@ "long_checkout": false, "py_versions": ["all"] }, + "pyanalyze": { + "cli_arguments": [], + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/quora/pyanalyze.git", + "long_checkout": false, + "py_versions": ["all"] + }, "pyramid": { "cli_arguments": [], "expect_formatting_changes": true, @@ -110,6 +117,13 @@ "long_checkout": false, "py_versions": ["all"] }, + "typeshed": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/python/typeshed.git", + "long_checkout": false, + "py_versions": ["all"] + }, "virtualenv": { "cli_arguments": [], "expect_formatting_changes": false, From 76e1602d60391cca1bb3b21a9ef6ed07a1bca17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 27 Apr 2021 17:58:39 +0200 Subject: [PATCH 33/81] Install primer.json (used by black-primer by default) with black (#2154) Fixes https://github.com/psf/black/issues/2153 --- CHANGES.md | 6 ++++++ setup.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d114c5136ff..37b5da6a40d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ ## Change Log +### Unreleased + +#### _Packaging_ + +- Install `primer.json` (used by `black-primer` by default) with black. (#2154) + ### 21.4b1 #### _Black_ diff --git a/setup.py b/setup.py index 0928c63afc2..f1792a46fe8 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,11 @@ def get_long_description() -> str: ext_modules=ext_modules, packages=["blackd", "black", "blib2to3", "blib2to3.pgen2", "black_primer"], package_dir={"": "src"}, - package_data={"blib2to3": ["*.txt"], "black": ["py.typed"]}, + package_data={ + "blib2to3": ["*.txt"], + "black": ["py.typed"], + "black_primer": ["primer.json"], + }, python_requires=">=3.6.2", zip_safe=False, install_requires=[ From 807a65f9d59d97a606575dc69110d7a5dfd98641 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 27 Apr 2021 14:00:17 -0700 Subject: [PATCH 34/81] Update discussion of AST safety check in README (#2159) --- CHANGES.md | 5 +++++ README.md | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 37b5da6a40d..a30668d33b2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,11 @@ ### Unreleased +#### _Black_ + +- Clarify that _Black_ may change the AST, especially when cleaning up docstrings. + (#2159) + #### _Packaging_ - Install `primer.json` (used by `black-primer` by default) with black. (#2154) diff --git a/README.md b/README.md index 8d6adb0567e..c7577fbbd04 100644 --- a/README.md +++ b/README.md @@ -238,8 +238,10 @@ change in the future**. That being said, no drastic stylistic changes are planne mostly responses to bug reports. Also, as a temporary safety measure, _Black_ will check that the reformatted code still -produces a valid AST that is equivalent to the original. This slows it down. If you're -feeling confident, use `--fast`. +produces a valid AST that is mostly equivalent to the original. This slows it down. If +you're feeling confident, use `--fast`. In a few contexts, Black does make changes to +the AST: it cleans up whitespace in docstrings, adds or removes parentheses in some +`del` statements, and may move around type comments. ## The _Black_ code style From ad1696422b7343a0cfea3c4b304e6acfa15d3cff Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 27 Apr 2021 14:16:35 -0700 Subject: [PATCH 35/81] Ignore inaccessible user config (#2158) Fixes #2157 --- CHANGES.md | 2 ++ src/black/__init__.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a30668d33b2..cae61f25b4e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ #### _Black_ +- Fix crash if the user configuration directory is inaccessible. (#2158) + - Clarify that _Black_ may change the AST, especially when cleaning up docstrings. (#2159) diff --git a/src/black/__init__.py b/src/black/__init__.py index 015ca415f1c..1c69cc41cdc 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -311,8 +311,17 @@ def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: if path_pyproject_toml.is_file(): return str(path_pyproject_toml) - path_user_pyproject_toml = find_user_pyproject_toml() - return str(path_user_pyproject_toml) if path_user_pyproject_toml.is_file() else None + try: + path_user_pyproject_toml = find_user_pyproject_toml() + return ( + str(path_user_pyproject_toml) + if path_user_pyproject_toml.is_file() + else None + ) + except PermissionError as e: + # We do not have access to the user-level config directory, so ignore it. + err(f"Ignoring user configuration directory due to {e!r}") + return None def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: From 04fd4432f6c6007e419d7174930569a75ea60fed Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 27 Apr 2021 20:10:44 -0700 Subject: [PATCH 36/81] Add automatic version tagging to Docker CI Pushes (#2132) * Add automatic version tagging to Docker Uploads - If the git comment has a tag, set that on the docker images pushed - If we don't have a tag, we just set `latest_non_release` * Add trigger on release creation too * Make prettier happy omn docker.yml --- .github/workflows/docker.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 09390130b3d..7eaf233ab9c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,8 @@ on: push: branches: - "master" + release: + types: created jobs: docker: @@ -24,13 +26,18 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Check + set version tag + run: + echo "GIT_TAG=$(git describe --candidates=0 --tags 2> /dev/null || echo + latest_non_release)" >> $GITHUB_ENV + - name: Build and push uses: docker/build-push-action@v2 with: context: . platforms: linux/amd64,linux/arm64 push: true - tags: pyfound/black:latest + tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 89a856d742d31054bc31a6ff7ffc6ad2d24377d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 28 Apr 2021 16:07:15 +0200 Subject: [PATCH 37/81] Remove useless shebangs in non-executable files (#2161) Such shebangs are only ever used if the file is executed directly, i.e.: $ /usr/lib/python3.9/site-packages/black_primer/cli.py But that doesn't work: $ /usr/lib/python3.9/site-packages/black_primer/cli.py bash: /usr/lib/python3.9/site-packages/black_primer/cli.py: Permission denied The lib file even has: "lib is a library, funnily enough" --- src/black_primer/cli.py | 2 -- src/black_primer/lib.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 5903adc72d4..b2d4159b1da 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # coding=utf8 import asyncio diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index 7999f6a3e36..ecc704f2cca 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import asyncio import errno import json From 682077c15e914fa661879003e186ffc90ac66dcc Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Wed, 28 Apr 2021 07:21:56 -0700 Subject: [PATCH 38/81] Update CHANGELOG for 21.4b2 release (#2162) --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cae61f25b4e..5b9f574a408 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ ## Change Log -### Unreleased +### 21.4b2 #### _Black_ From b39999da7f451c285befac217f1f9a685774b34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 28 Apr 2021 16:48:04 +0200 Subject: [PATCH 39/81] Elaborate on what AST changes Black might perform --- CHANGES.md | 5 +++-- README.md | 10 +++++----- docs/the_black_code_style.md | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5b9f574a408..6675c484c75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,8 +6,9 @@ - Fix crash if the user configuration directory is inaccessible. (#2158) -- Clarify that _Black_ may change the AST, especially when cleaning up docstrings. - (#2159) +- Clarify + [circumstances](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) + in which _Black_ may change the AST (#2159) #### _Packaging_ diff --git a/README.md b/README.md index c7577fbbd04..41cea761df5 100644 --- a/README.md +++ b/README.md @@ -237,11 +237,11 @@ is that **until the formatter becomes stable, you should expect some formatting change in the future**. That being said, no drastic stylistic changes are planned, mostly responses to bug reports. -Also, as a temporary safety measure, _Black_ will check that the reformatted code still -produces a valid AST that is mostly equivalent to the original. This slows it down. If -you're feeling confident, use `--fast`. In a few contexts, Black does make changes to -the AST: it cleans up whitespace in docstrings, adds or removes parentheses in some -`del` statements, and may move around type comments. +Also, as a safety measure which slows down processing, _Black_ will check that the +reformatted code still produces a valid AST that is effectively equivalent to the +original (see the +[Pragmatism](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) +section for details). If you're feeling confident, use `--fast`. ## The _Black_ code style diff --git a/docs/the_black_code_style.md b/docs/the_black_code_style.md index 67ccb8aab11..39a452ff9a8 100644 --- a/docs/the_black_code_style.md +++ b/docs/the_black_code_style.md @@ -464,3 +464,32 @@ exception to this rule is r-strings. It turns out that the very popular default by (among others) GitHub and Visual Studio Code, differentiates between r-strings and R-strings. The former are syntax highlighted as regular expressions while the latter are treated as true raw strings with no special semantics. + +### AST before and after formatting + +When run with `--safe`, _Black_ checks that the code before and after is semantically +equivalent. This check is done by comparing the AST of the source with the AST of the +target. There are three limited cases in which the AST does differ: + +1. _Black_ cleans up leading and trailing whitespace of docstrings, re-indenting them if + needed. It's been one of the most popular user-reported features for the formatter to + fix whitespace issues with docstrings. While the result is technically an AST + difference, due to the various possibilities of forming docstrings, all realtime use + of docstrings that we're aware of sanitizes indentation and leading/trailing + whitespace anyway. + +2. _Black_ manages optional parentheses for some statements. In the case of the `del` + statement, presence of wrapping parentheses or lack of thereof changes the resulting + AST but is semantically equivalent in the interpreter. + +3. _Black_ might move comments around, which includes type comments. Those are part of + the AST as of Python 3.8. While the tool implements a number of special cases for + those comments, there is no guarantee they will remain where they were in the source. + Note that this doesn't change runtime behavior of the source code. + +To put things in perspective, the code equivalence check is a feature of _Black_ which +other formatters don't implement at all. It is of crucial importance to us to ensure +code behaves the way it did before it got reformatted. We treat this as a feature and +there are no plans to relax this in the future. The exceptions enumerated above stem +from either user feedback or implementation details of the tool. In each case we made +due diligence to ensure that the AST divergence is of no practical consequence. From 00491e1dcb19a795867249471ca5bb95efcd8cd3 Mon Sep 17 00:00:00 2001 From: Kaleb Barrett Date: Sat, 1 May 2021 13:47:59 -0500 Subject: [PATCH 40/81] Add ability to pass posargs to pytest run in tox.ini (#2173) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 71b02e920c7..affc3c9876a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = commands = pip install -e .[d,python2] coverage erase - coverage run -m pytest tests + coverage run -m pytest tests {posargs} coverage report [testenv:fuzz] From 24bd6b983ac459eabd9352eb816be53e7f447812 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 1 May 2021 22:17:20 +0300 Subject: [PATCH 41/81] Tox has been formatted with Black 21.4b0 (#2175) --- src/black_primer/primer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 75469ad815b..5017a5625a3 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -112,7 +112,7 @@ }, "tox": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/tox-dev/tox.git", "long_checkout": false, "py_versions": ["all"] From 35e8d1560d68e8113ff926a9f832582f8f4a694f Mon Sep 17 00:00:00 2001 From: Bryan Forbes Date: Sun, 2 May 2021 07:48:54 -0500 Subject: [PATCH 42/81] Set `is_pyi` if `stdin_filename` ends with `.pyi` (#2169) Fixes #2167 Co-authored-by: Jelle Zijlstra --- CHANGES.md | 6 ++++++ src/black/__init__.py | 2 ++ tests/test_black.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6675c484c75..321f8c083b2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ ## Change Log +### Unreleased + +#### _Black_ + +- Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) + ### 21.4b2 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 1c69cc41cdc..49d088b531d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -755,6 +755,8 @@ def reformat_one( is_stdin = False if is_stdin: + if src.suffix == ".pyi": + mode = replace(mode, is_pyi=True) if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: diff --git a/tests/test_black.py b/tests/test_black.py index c643f27cacf..43368d4bbe9 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1578,8 +1578,34 @@ def test_reformat_one_with_stdin_filename(self) -> None: mode=DEFAULT_MODE, report=report, ) - fsts.assert_called_once() - # __BLACK_STDIN_FILENAME__ should have been striped + fsts.assert_called_once_with( + fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE + ) + # __BLACK_STDIN_FILENAME__ should have been stripped + report.done.assert_called_with(expected, black.Changed.YES) + + def test_reformat_one_with_stdin_filename_pyi(self) -> None: + with patch( + "black.format_stdin_to_stdout", + return_value=lambda *args, **kwargs: black.Changed.YES, + ) as fsts: + report = MagicMock() + p = "foo.pyi" + path = Path(f"__BLACK_STDIN_FILENAME__{p}") + expected = Path(p) + black.reformat_one( + path, + fast=True, + write_back=black.WriteBack.YES, + mode=DEFAULT_MODE, + report=report, + ) + fsts.assert_called_once_with( + fast=True, + write_back=black.WriteBack.YES, + mode=replace(DEFAULT_MODE, is_pyi=True), + ) + # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) def test_reformat_one_with_stdin_and_existing_path(self) -> None: @@ -1603,7 +1629,7 @@ def test_reformat_one_with_stdin_and_existing_path(self) -> None: report=report, ) fsts.assert_called_once() - # __BLACK_STDIN_FILENAME__ should have been striped + # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) def test_gitignore_exclude(self) -> None: From a669b64091c149132f4e234abd230f7b33d692e8 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 3 May 2021 14:58:17 -0700 Subject: [PATCH 43/81] primer: Renable pandas (#2185) - It no longer crashes black so we should test on it's code - Update django reason to name the file causing error - Seems it has a syntax error on purpose --- src/black_primer/primer.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 5017a5625a3..137d075c68e 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -30,7 +30,7 @@ "py_versions": ["all"] }, "django": { - "disabled_reason": "black --check --diff returned 123 on two files", + "disabled_reason": "black --check --diff returned 123 on tests_syntax_error.py", "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, @@ -53,8 +53,6 @@ "py_versions": ["all"] }, "pandas": { - "disabled_reason": "black --check --diff returned 123 on one file", - "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", From a18c7bc09942951e93cbd142fb7384aa2af18951 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 01:44:40 -0700 Subject: [PATCH 44/81] primer: Add `--no-diff` option (#2187) - Allow runs with no code diff output - This is handy for reducing output to see which file is erroring Test: - Edit config for 'channels' to expect no changes and run with `--no-diff` and see no diff output --- CHANGES.md | 1 + src/black_primer/cli.py | 15 ++++++++++++++- src/black_primer/lib.py | 25 +++++++++++++++++++++---- tests/test_primer.py | 1 + 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 321f8c083b2..7741d92772d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ #### _Black_ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) +- Add `--no-diff` to black-primer to suppress formatting changes (#2187) ### 21.4b2 diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index b2d4159b1da..00b54d6511f 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -39,6 +39,7 @@ async def async_main( debug: bool, keep: bool, long_checkouts: bool, + no_diff: bool, rebase: bool, workdir: str, workers: int, @@ -54,7 +55,13 @@ async def async_main( try: ret_val = await lib.process_queue( - config, work_path, workers, keep, long_checkouts, rebase + config, + work_path, + workers, + keep, + long_checkouts, + rebase, + no_diff, ) return int(ret_val) finally: @@ -95,6 +102,12 @@ async def async_main( show_default=True, help="Pull big projects to test", ) +@click.option( + "--no-diff", + is_flag=True, + show_default=True, + help="Disable showing source file changes in black output", +) @click.option( "-R", "--rebase", diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index ecc704f2cca..aba694b0e60 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -112,13 +112,20 @@ def analyze_results(project_count: int, results: Results) -> int: async def black_run( - repo_path: Path, project_config: Dict[str, Any], results: Results + repo_path: Path, + project_config: Dict[str, Any], + results: Results, + no_diff: bool = False, ) -> None: """Run Black and record failures""" cmd = [str(which(BLACK_BINARY))] if "cli_arguments" in project_config and project_config["cli_arguments"]: cmd.extend(*project_config["cli_arguments"]) - cmd.extend(["--check", "--diff", "."]) + cmd.append("--check") + if no_diff: + cmd.append(".") + else: + cmd.extend(["--diff", "."]) with TemporaryDirectory() as tmp_path: # Prevent reading top-level user configs by manipulating envionment variables @@ -246,6 +253,7 @@ async def project_runner( long_checkouts: bool = False, rebase: bool = False, keep: bool = False, + no_diff: bool = False, ) -> None: """Check out project and run Black on it + record result""" loop = asyncio.get_event_loop() @@ -284,7 +292,7 @@ async def project_runner( repo_path = await git_checkout_or_rebase(work_path, project_config, rebase) if not repo_path: continue - await black_run(repo_path, project_config, results) + await black_run(repo_path, project_config, results, no_diff) if not keep: LOG.debug(f"Removing {repo_path}") @@ -303,6 +311,7 @@ async def process_queue( keep: bool = False, long_checkouts: bool = False, rebase: bool = False, + no_diff: bool = False, ) -> int: """ Process the queue with X workers and evaluate results @@ -330,7 +339,15 @@ async def process_queue( await asyncio.gather( *[ project_runner( - i, config, queue, work_path, results, long_checkouts, rebase, keep + i, + config, + queue, + work_path, + results, + long_checkouts, + rebase, + keep, + no_diff, ) for i in range(workers) ] diff --git a/tests/test_primer.py b/tests/test_primer.py index a8ad8a7c5af..8bfecd61a57 100644 --- a/tests/test_primer.py +++ b/tests/test_primer.py @@ -198,6 +198,7 @@ def test_async_main(self) -> None: "rebase": False, "workdir": str(work_dir), "workers": 69, + "no_diff": False, } with patch("black_primer.cli.lib.process_queue", return_zero): return_val = loop.run_until_complete(cli.async_main(**args)) From e42f9921e291790202147c1e3dc1a3df7036c652 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 4 May 2021 04:46:46 -0400 Subject: [PATCH 45/81] Detect `'@' dotted_name '(' ')' NEWLINE` as a simple decorator (#2182) Previously the RELAXED_DECORATOR detection would be falsely True on that example. The problem was that an argument-less parentheses pair didn't pass the `is_simple_decorator_trailer` check even it should. OTOH a parentheses pair containing an argument or more passed as expected. --- CHANGES.md | 3 +++ src/black/__init__.py | 7 +++++++ tests/data/decorators.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7741d92772d..00d6782dc6b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,9 @@ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) - Add `--no-diff` to black-primer to suppress formatting changes (#2187) +- Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are + being called but with no arguments (#2182) + ### 21.4b2 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 49d088b531d..cf257876c03 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5761,6 +5761,13 @@ def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool: and node.children[0].type == token.DOT and node.children[1].type == token.NAME ) + # last trailer can be an argument-less parentheses pair + or ( + last + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) # last trailer can be arguments or ( last diff --git a/tests/data/decorators.py b/tests/data/decorators.py index acfad51fcb8..a0f38ca7b9d 100644 --- a/tests/data/decorators.py +++ b/tests/data/decorators.py @@ -13,6 +13,12 @@ def f(): ## +@decorator() +def f(): + ... + +## + @decorator(arg) def f(): ... From 204f76e0c02a12aeccb3ab708fe41f7e95435a29 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+KotlinIsland@users.noreply.github.com> Date: Tue, 4 May 2021 18:47:22 +1000 Subject: [PATCH 46/81] add test configurations that don't contain python2 optional install (#2190) add test for negative scenario: formatting python2 code tag python2 only tests Co-authored-by: KotlinIsland --- pyproject.toml | 3 +++ tests/test_black.py | 30 ++++++++++++++++++++++++++++++ tests/test_format.py | 16 +++++++++++++--- tox.ini | 6 ++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7f632f2839d..ca75f8f92ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,6 @@ extend-exclude = ''' [build-system] requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = ['python2', "without_python2"] \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index 43368d4bbe9..7d855cab3d3 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -24,6 +24,7 @@ Iterator, TypeVar, ) +import pytest import unittest from unittest.mock import patch, MagicMock @@ -459,6 +460,34 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) + @pytest.mark.without_python2 + def test_python2_should_fail_without_optional_install(self) -> None: + # python 3.7 and below will install typed-ast and will be able to parse Python 2 + if sys.version_info < (3, 8): + return + source = "x = 1234l" + tmp_file = Path(black.dump_to_file(source)) + try: + runner = BlackRunner() + result = runner.invoke(black.main, [str(tmp_file)]) + self.assertEqual(result.exit_code, 123) + finally: + os.unlink(tmp_file) + actual = ( + runner.stderr_bytes.decode() + .replace("\n", "") + .replace("\\n", "") + .replace("\\r", "") + .replace("\r", "") + ) + msg = ( + "The requested source code has invalid Python 3 syntax." + "If you are trying to format Python 2 files please reinstall Black" + " with the 'python2' extra: `python3 -m pip install black[python2]`." + ) + self.assertIn(msg, actual) + + @pytest.mark.python2 @patch("black.dump_to_file", dump_to_stderr) def test_python2_print_function(self) -> None: source, expected = read_data("python2_print_function") @@ -1971,6 +2000,7 @@ def test_bpo_2142_workaround(self) -> None: actual = diff_header.sub(DETERMINISTIC_HEADER, actual) self.assertEqual(actual, expected) + @pytest.mark.python2 def test_docstring_reformat_for_py27(self) -> None: """ Check that stripping trailing whitespace from Python 2 docstrings diff --git a/tests/test_format.py b/tests/test_format.py index e1335aedf43..78f2b558a4f 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,6 +1,7 @@ from unittest.mock import patch import black +import pytest from parameterized import parameterized from tests.util import ( @@ -46,9 +47,6 @@ "function2", "function_trailing_comma", "import_spacing", - "numeric_literals_py2", - "python2", - "python2_unicode_literals", "remove_parens", "slices", "string_prefixes", @@ -56,6 +54,12 @@ "tupleassign", ] +SIMPLE_CASES_PY2 = [ + "numeric_literals_py2", + "python2", + "python2_unicode_literals", +] + EXPERIMENTAL_STRING_PROCESSING_CASES = [ "cantfit", "comments7", @@ -86,6 +90,12 @@ class TestSimpleFormat(BlackBaseTestCase): + @parameterized.expand(SIMPLE_CASES_PY2) + @pytest.mark.python2 + @patch("black.dump_to_file", dump_to_stderr) + def test_simple_format_py2(self, filename: str) -> None: + self.check_file(filename, DEFAULT_MODE) + @parameterized.expand(SIMPLE_CASES) @patch("black.dump_to_file", dump_to_stderr) def test_simple_format(self, filename: str) -> None: diff --git a/tox.ini b/tox.ini index affc3c9876a..cbb0f75d145 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,11 @@ skip_install = True deps = -r{toxinidir}/test_requirements.txt commands = - pip install -e .[d,python2] + pip install -e .[d] coverage erase - coverage run -m pytest tests {posargs} + coverage run -m pytest tests -m "not python2" {posargs} + pip install -e .[d,python2] + coverage run -m pytest tests -m "not without_python2" {posargs} coverage report [testenv:fuzz] From 5918a016ff82e5fa12097d07b1624a89ec4e60ac Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 4 May 2021 04:47:59 -0400 Subject: [PATCH 47/81] Drop Travis CI and migrate Coveralls (#2186) Travis CI for Open Source is shutting down in a few weeks so the queue for jobs is insane due to lower resources. I'm 99.99% sure we don't need it as our Test, Lint, Docs, Upload / Package, Primer, and Fuzz workflows are all on GitHub Actions. So even though we *can* migrate to the .com version with its 1000 free Linux minutes(?), I don't think we need to. more information here: - https://blog.travis-ci.com/oss-announcement - https://blog.travis-ci.com/2020-11-02-travis-ci-new-billing - https://docs.travis-ci.com/user/migrate/open-source-repository-migration This commit does the following: - delete the Travis CI configuration - add to the GHA test workflows so coverage continues to be recorded - tweaked coverage configuration so this wouldn't break - remove any references to Travis CI in the docs (i.e. readme + sphinx docs) Regarding the Travis CI to GitHub Actions Coveralls transition, the official action doesn't support the coverage files produced by coverage.py unfornately. Also no, I don't really know what I am doing so don't @ me if this breaks :p (well you can, but don't expect me to be THAT useful). The Coveralls setup has two downfalls AFAIK: - Only Linux runs are used because AndreMiras/coveralls-python-action only supports Linux. Although this isn't a big issue since the Travis Coveralls configuration only used Linux data too. - Pull requests from an internal branch (i.e. one on psf/black) will be marked as a push coverage build by Coveralls since our anti-duplicate- workflows system runs under the push even for such cases. --- .coveragerc | 3 +++ .github/workflows/test.yml | 31 +++++++++++++++++++++++++++++++ .travis.yml | 31 ------------------------------- README.md | 1 - docs/conf.py | 1 - 5 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.coveragerc b/.coveragerc index 32a8da521ba..f05041ca1dc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,6 @@ omit = src/blib2to3/* tests/data/* */site-packages/* + +[run] +relative_files = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bec769064c2..03adc7f0bea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,34 @@ jobs: - name: Unit tests run: | tox -e py + + - name: Publish coverage to Coveralls + # If pushed / is a pull request against main repo AND + # we're running on Linux (this action only supports Linux) + if: + ((github.event_name == 'push' && github.repository == 'psf/black') || + github.event.pull_request.base.repo.full_name == 'psf/black') && matrix.os == + 'ubuntu-latest' + + uses: AndreMiras/coveralls-python-action@v20201129 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel: true + flag-name: py${{ matrix.python-version }}-${{ matrix.os }} + debug: true + + coveralls-finish: + needs: build + # If pushed / is a pull request against main repo + if: + (github.event_name == 'push' && github.repository == 'psf/black') || + github.event.pull_request.base.repo.full_name == 'psf/black' + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Coveralls finished + uses: AndreMiras/coveralls-python-action@v20201129 + with: + parallel-finished: true + debug: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e782272e813..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python -cache: - pip: true - directories: - - $HOME/.cache/pre-commit -env: - - TEST_CMD="tox -e py" -install: - - pip install coverage coveralls pre-commit tox - - pip install -e '.[d]' -script: - - $TEST_CMD -after_success: - - coveralls -notifications: - on_success: change - on_failure: always -matrix: - include: - - name: "lint" - python: 3.7 - env: - - TEST_CMD="pre-commit run --all-files --show-diff-on-failure" - - name: "3.6" - python: 3.6 - - name: "3.7" - python: 3.7 - - name: "3.8" - python: 3.8 - - name: "3.9" - python: 3.9 diff --git a/README.md b/README.md index 41cea761df5..696bfa7e064 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@

The Uncompromising Code Formatter

-Build Status Actions Status Actions Status Documentation Status diff --git a/docs/conf.py b/docs/conf.py index ec6aad9a27b..9e03d05e937 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -314,7 +314,6 @@ def process_sections( "show_powered_by": True, "fixed_sidebar": True, "logo": "logo2.png", - "travis_button": True, } From 0c60ccc06646030bd48d4e160c6aae755307c2d9 Mon Sep 17 00:00:00 2001 From: reka <382113+reka@users.noreply.github.com> Date: Tue, 4 May 2021 10:48:59 +0200 Subject: [PATCH 48/81] compatible isort config: mention profile first (#2180) Change the order of possible ways to configure isort: 1. using the profile black 2. custom configuration Formats section: change the examples to use the profile black Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- docs/compatible_configs.md | 50 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/compatible_configs.md b/docs/compatible_configs.md index a4f83ee472f..de81769f72d 100644 --- a/docs/compatible_configs.md +++ b/docs/compatible_configs.md @@ -19,7 +19,24 @@ Compatible configuration files can be _Black_ also formats imports, but in a different way from isort's defaults which leads to conflicting changes. -### Configuration +### Profile + +Since version 5.0.0, isort supports +[profiles](https://pycqa.github.io/isort/docs/configuration/profiles/) to allow easy +interoperability with common code styles. You can set the black profile in any of the +[config files](https://pycqa.github.io/isort/docs/configuration/config_files/) supported +by isort. Below, an example for `pyproject.toml`: + +```toml +[tool.isort] +profile = "black" +``` + +### Custom Configuration + +If you're using an isort version that is older than 5.0.0 or you have some custom +configuration for _Black_, you can tweak your isort configuration to make it compatible +with _Black_. Below, an example for `.isort.cfg`: ``` multi_line_output = 3 @@ -72,9 +89,6 @@ works the same as with _Black_. **Please note** `ensure_newline_before_comments = True` only works since isort >= 5 but does not break older versions so you can keep it if you are running previous versions. -If only isort >= 5 is used you can add `profile = black` instead of all the options -since [profiles](https://timothycrosley.github.io/isort/docs/configuration/profiles/) -are available and do the configuring for you. ### Formats @@ -83,12 +97,7 @@ are available and do the configuring for you. ```cfg [settings] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` @@ -98,12 +107,7 @@ line_length = 88 ```cfg [isort] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` @@ -113,12 +117,7 @@ line_length = 88 ```toml [tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 88 +profile = 'black' ``` @@ -128,12 +127,7 @@ line_length = 88 ```ini [*.py] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` From 267bc5dde9f4e5e4b6dacdf79cf1688ffe9b7715 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 4 May 2021 14:41:04 +0300 Subject: [PATCH 49/81] Use pre-commit/action to simplify CI (#2191) --- .github/workflows/lint.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2480a5ec131..51f6d02e2e6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,8 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install --upgrade pre-commit python -m pip install -e '.[d]' - name: Lint - run: pre-commit run --all-files --show-diff-on-failure + uses: pre-commit/action@v2.0.2 From d8a034f9b69b3a214e01985e741c4418ae562e2e Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 11:07:08 -0700 Subject: [PATCH 50/81] Update CHANGES.md for 21.5b0 release (#2192) * Update CHANGES.md for 21.5b0 release * Make prettier happy --- CHANGES.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 00d6782dc6b..becf69904fb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,15 +1,17 @@ ## Change Log -### Unreleased +### 21.5b0 #### _Black_ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) -- Add `--no-diff` to black-primer to suppress formatting changes (#2187) - - Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are being called but with no arguments (#2182) +#### _Black-Primer_ + +- Add `--no-diff` to black-primer to suppress formatting changes (#2187) + ### 21.4b2 #### _Black_ From 14c76e89716b5b53c97ece80bb935ea956b7dd89 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 12:49:20 -0700 Subject: [PATCH 51/81] Disable pandas while we look into #2193 (#2195) --- src/black_primer/primer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 137d075c68e..78c1e2acdef 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -53,6 +53,8 @@ "py_versions": ["all"] }, "pandas": { + "disabled_reason": "black-primer runs failing on Pandas - #2193", + "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", From 07c8812937cf75ac5bc7ceac07ef5ea383f10f2f Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Wed, 5 May 2021 08:33:23 -0700 Subject: [PATCH 52/81] Enable ` --experimental-string-processing` on most primer projects (#2184) * Enable ` --experimental-string-processing` on all primer projects - We want to make this default so need to test it more - Fixed splat/star bug in extending black args for each project * Disable sqlalchemy due to crash --- src/black_primer/lib.py | 2 +- src/black_primer/primer.json | 47 ++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index aba694b0e60..3ce383f17ce 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -120,7 +120,7 @@ async def black_run( """Run Black and record failures""" cmd = [str(which(BLACK_BINARY))] if "cli_arguments" in project_config and project_config["cli_arguments"]: - cmd.extend(*project_config["cli_arguments"]) + cmd.extend(project_config["cli_arguments"]) cmd.append("--check") if no_diff: cmd.append(".") diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 78c1e2acdef..76ed4820487 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -2,28 +2,28 @@ "configuration_format_version": 20200509, "projects": { "aioexabgp": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] }, "attrs": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-attrs/attrs.git", "long_checkout": false, "py_versions": ["all"] }, "bandersnatch": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] }, "channels": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/django/channels.git", "long_checkout": false, @@ -32,22 +32,22 @@ "django": { "disabled_reason": "black --check --diff returned 123 on tests_syntax_error.py", "disabled": true, - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/django/django.git", "long_checkout": false, "py_versions": ["all"] }, "flake8-bugbear": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", "long_checkout": false, "py_versions": ["all"] }, "hypothesis": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/HypothesisWorks/hypothesis.git", "long_checkout": false, "py_versions": ["all"] @@ -55,55 +55,56 @@ "pandas": { "disabled_reason": "black-primer runs failing on Pandas - #2193", "disabled": true, - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", "long_checkout": false, "py_versions": ["all"] }, "pillow": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-pillow/Pillow.git", "long_checkout": false, "py_versions": ["all"] }, "poetry": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-poetry/poetry.git", "long_checkout": false, "py_versions": ["all"] }, "pyanalyze": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/quora/pyanalyze.git", "long_checkout": false, "py_versions": ["all"] }, "pyramid": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/Pylons/pyramid.git", "long_checkout": false, "py_versions": ["all"] }, "ptr": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/facebookincubator/ptr.git", "long_checkout": false, "py_versions": ["all"] }, "pytest": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pytest-dev/pytest.git", "long_checkout": false, "py_versions": ["all"] }, "sqlalchemy": { + "no_cli_args_reason": "breaks black with new string parsing - #2188", "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git", @@ -111,28 +112,28 @@ "py_versions": ["all"] }, "tox": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/tox-dev/tox.git", "long_checkout": false, "py_versions": ["all"] }, "typeshed": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python/typeshed.git", "long_checkout": false, "py_versions": ["all"] }, "virtualenv": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/virtualenv.git", "long_checkout": false, "py_versions": ["all"] }, "warehouse": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/warehouse.git", "long_checkout": false, From 5316836393682c6ec6a05d69c549d8167f46d8f6 Mon Sep 17 00:00:00 2001 From: Shota Ray Imaki Date: Thu, 6 May 2021 11:25:43 +0900 Subject: [PATCH 53/81] Simplify GitHub Action entrypoint (#2119) This commit simplifies entrypoint.sh for GitHub Actions by removing duplication of args and black_args (cf. #1909). The reason why #1909 uses the input id black_args is to avoid an overlap with args, but this naming seems redundant. So let me suggest option and src, which are consistent with CLI. Backward compatibility is guaranteed; Users can still use black_args as well. Commit history pre-merge: * Simplify GitHub Action entrypoint (#1909) * Fix prettier * Emit a warning message when `black_args` is used This deprecation should be visible in GitHub Action's UI now. Co-authored-by: Shota Ray Imaki Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- README.md | 14 ++++++++------ action.yml | 12 +++++++++++- action/entrypoint.sh | 21 ++++++--------------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 696bfa7e064..6443569d0d0 100644 --- a/README.md +++ b/README.md @@ -425,15 +425,17 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: psf/black@stable - with: - args: ". --check" ``` -### Inputs +You may use `options` (Default is `'--check --diff'`) and `src` (Default is `'.'`) as +follows: -#### `black_args` - -**optional**: Black input arguments. Defaults to `. --check --diff`. +```yaml +- uses: psf/black@stable + with: + options: "--check --verbose" + src: "./src" +``` ## Ignoring unmodified files diff --git a/action.yml b/action.yml index 59b16a9fb6c..827e971801b 100644 --- a/action.yml +++ b/action.yml @@ -2,8 +2,18 @@ name: "Black" description: "The uncompromising Python code formatter." author: "Łukasz Langa and contributors to Black" inputs: + options: + description: + "Options passed to black. Use `black --help` to see available options. Default: + '--check'" + required: false + default: "--check --diff" + src: + description: "Source to run black. Default: '.'" + required: false + default: "." black_args: - description: "Black input arguments." + description: "[DEPRECATED] Black input arguments." required: false default: "" branding: diff --git a/action/entrypoint.sh b/action/entrypoint.sh index 50f9472cc5f..fc66e24f53a 100755 --- a/action/entrypoint.sh +++ b/action/entrypoint.sh @@ -1,17 +1,8 @@ -#!/bin/bash -set -e +#!/bin/bash -e -# If no arguments are given use current working directory -black_args=(".") -if [ "$#" -eq 0 ] && [ "${INPUT_BLACK_ARGS}" != "" ]; then - black_args+=(${INPUT_BLACK_ARGS}) -elif [ "$#" -ne 0 ] && [ "${INPUT_BLACK_ARGS}" != "" ]; then - black_args+=($* ${INPUT_BLACK_ARGS}) -elif [ "$#" -ne 0 ] && [ "${INPUT_BLACK_ARGS}" == "" ]; then - black_args+=($*) -else - # Default (if no args provided). - black_args+=("--check" "--diff") -fi +if [ -n $INPUT_BLACK_ARGS ]; then + echo '::warning::Input `with.black_args` is deprecated. Use `with.options` and `with.src` instead.' + black $INPUT_BLACK_ARGS + exit $? -sh -c "black . ${black_args[*]}" +black $INPUT_OPTIONS $INPUT_SRC From 4b7b5ed5b8630a906e2ef3a405134131651a251e Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Thu, 6 May 2021 15:21:13 -0400 Subject: [PATCH 54/81] Fix broken Action entrypoint (#2202) --- action/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action/entrypoint.sh b/action/entrypoint.sh index fc66e24f53a..30bf4eb688f 100755 --- a/action/entrypoint.sh +++ b/action/entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash -e -if [ -n $INPUT_BLACK_ARGS ]; then +if [ -n "$INPUT_BLACK_ARGS" ]; then echo '::warning::Input `with.black_args` is deprecated. Use `with.options` and `with.src` instead.' black $INPUT_BLACK_ARGS exit $? +fi black $INPUT_OPTIONS $INPUT_SRC From 1fe2efd8573a63ffc76c69320720d349b21897da Mon Sep 17 00:00:00 2001 From: Kaleb Barrett Date: Fri, 7 May 2021 07:54:21 -0500 Subject: [PATCH 55/81] Do not use gitignore if explicitly passing excludes (#2170) Closes #2164. Changes behavior of how .gitignore is handled. With this change, the rules in .gitignore are only used as a fallback if no exclusion rule is explicitly passed on the command line or in pyproject.toml. Previously they were used regardless if explicit exclusion rules were specified, preventing any overriding of .gitignore rules. Those that depend only on .gitignore for their exclusion rules will not be affected. Those that use both .gitignore and exclude will find that exclude will act more like actually specifying exclude and not just another extra-excludes. If the previous behavior was desired, they should move their rules from exclude to extra-excludes. --- CHANGES.md | 3 ++ src/black/__init__.py | 25 ++++++++++------- tests/data/include_exclude_tests/.gitignore | 1 + .../data/include_exclude_tests/pyproject.toml | 3 ++ tests/test_black.py | 28 +++++++++++++++++++ 5 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 tests/data/include_exclude_tests/.gitignore create mode 100644 tests/data/include_exclude_tests/pyproject.toml diff --git a/CHANGES.md b/CHANGES.md index becf69904fb..b81e6eb4411 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,9 @@ [circumstances](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) in which _Black_ may change the AST (#2159) +- Allow `.gitignore` rules to be overridden by specifying `exclude` in `pyproject.toml` + or on the command line. (#2170) + #### _Packaging_ - Install `primer.json` (used by `black-primer` by default) with black. (#2154) diff --git a/src/black/__init__.py b/src/black/__init__.py index cf257876c03..e47aa21d435 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -492,15 +492,15 @@ def validate_regex( @click.option( "--exclude", type=str, - default=DEFAULT_EXCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" " excluded on recursive searches. An empty value means no paths are excluded." " Use forward slashes for directories on all platforms (Windows, too)." - " Exclusions are calculated first, inclusions later." + " Exclusions are calculated first, inclusions later. [default:" + f" {DEFAULT_EXCLUDES}]" ), - show_default=True, + show_default=False, ) @click.option( "--extend-exclude", @@ -587,7 +587,7 @@ def main( quiet: bool, verbose: bool, include: Pattern, - exclude: Pattern, + exclude: Optional[Pattern], extend_exclude: Optional[Pattern], force_exclude: Optional[Pattern], stdin_filename: Optional[str], @@ -662,7 +662,7 @@ def get_sources( quiet: bool, verbose: bool, include: Pattern[str], - exclude: Pattern[str], + exclude: Optional[Pattern[str]], extend_exclude: Optional[Pattern[str]], force_exclude: Optional[Pattern[str]], report: "Report", @@ -673,7 +673,12 @@ def get_sources( root = find_project_root(src) sources: Set[Path] = set() path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx) - gitignore = get_gitignore(root) + + if exclude is None: + exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) + gitignore = get_gitignore(root) + else: + gitignore = None for s in src: if s == "-" and stdin_filename: @@ -6215,12 +6220,12 @@ def path_is_excluded( def gen_python_files( paths: Iterable[Path], root: Path, - include: Optional[Pattern[str]], + include: Pattern[str], exclude: Pattern[str], extend_exclude: Optional[Pattern[str]], force_exclude: Optional[Pattern[str]], report: "Report", - gitignore: PathSpec, + gitignore: Optional[PathSpec], ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, @@ -6236,8 +6241,8 @@ def gen_python_files( if normalized_path is None: continue - # First ignore files matching .gitignore - if gitignore.match_file(normalized_path): + # First ignore files matching .gitignore, if passed + if gitignore is not None and gitignore.match_file(normalized_path): report.path_ignored(child, "matches the .gitignore file content") continue diff --git a/tests/data/include_exclude_tests/.gitignore b/tests/data/include_exclude_tests/.gitignore new file mode 100644 index 00000000000..91f34560522 --- /dev/null +++ b/tests/data/include_exclude_tests/.gitignore @@ -0,0 +1 @@ +dont_exclude/ diff --git a/tests/data/include_exclude_tests/pyproject.toml b/tests/data/include_exclude_tests/pyproject.toml new file mode 100644 index 00000000000..9ba7ec26980 --- /dev/null +++ b/tests/data/include_exclude_tests/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/tests/test_black.py b/tests/test_black.py index 7d855cab3d3..9b2bfcd740e 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1418,6 +1418,32 @@ def test_include_exclude(self) -> None: ) self.assertEqual(sorted(expected), sorted(sources)) + def test_gitingore_used_as_default(self) -> None: + path = Path(THIS_DIR / "data" / "include_exclude_tests") + include = re.compile(r"\.pyi?$") + extend_exclude = re.compile(r"/exclude/") + src = str(path / "b/") + report = black.Report() + expected: List[Path] = [ + path / "b/.definitely_exclude/a.py", + path / "b/.definitely_exclude/a.pyi", + ] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude=None, + extend_exclude=extend_exclude, + force_exclude=None, + report=report, + stdin_filename=None, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) def test_exclude_for_issue_1572(self) -> None: # Exclude shouldn't touch files that were explicitly given to Black through the @@ -1705,6 +1731,8 @@ def test_empty_include(self) -> None: Path(path / "b/.definitely_exclude/a.pie"), Path(path / "b/.definitely_exclude/a.py"), Path(path / "b/.definitely_exclude/a.pyi"), + Path(path / ".gitignore"), + Path(path / "pyproject.toml"), ] this_abs = THIS_DIR.resolve() sources.extend( From e4b4fb02b91e0f5a60a9678604653aecedff513b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 7 May 2021 15:03:13 +0200 Subject: [PATCH 56/81] Use optional tests for "no_python2" to simplify local testing (#2203) --- pyproject.toml | 5 +- tests/conftest.py | 1 + tests/optional.py | 119 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_black.py | 10 ++-- tox.ini | 4 +- 5 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/optional.py diff --git a/pyproject.toml b/pyproject.toml index ca75f8f92ef..e89cc7a6c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,4 +27,7 @@ requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -markers = ['python2', "without_python2"] \ No newline at end of file +# Option below requires `tests/optional.py` +optional-tests = [ + "no_python2: run when `python2` extra NOT installed", +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000000..67517268d1b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["tests.optional"] diff --git a/tests/optional.py b/tests/optional.py new file mode 100644 index 00000000000..e12b94cd29e --- /dev/null +++ b/tests/optional.py @@ -0,0 +1,119 @@ +""" +Allows configuring optional test markers in config, see pyproject.toml. + +Run optional tests with `pytest --run-optional=...`. + +Mark tests to run only if an optional test ISN'T selected by prepending the mark with +"no_". + +You can specify a "no_" prefix straight in config, in which case you can mark tests +to run when this tests ISN'T selected by omitting the "no_" prefix. + +Specifying the name of the default behavior in `--run-optional=` is harmless. + +Adapted from https://pypi.org/project/pytest-optional-tests/, (c) 2019 Reece Hart +""" + +from functools import lru_cache +import itertools +import logging +import re +from typing import FrozenSet, List, Set, TYPE_CHECKING + +import pytest +from _pytest.store import StoreKey + +log = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from _pytest.config.argparsing import Parser + from _pytest.config import Config + from _pytest.mark.structures import MarkDecorator + from _pytest.nodes import Node + + +ALL_POSSIBLE_OPTIONAL_MARKERS = StoreKey[FrozenSet[str]]() +ENABLED_OPTIONAL_MARKERS = StoreKey[FrozenSet[str]]() + + +def pytest_addoption(parser: "Parser") -> None: + group = parser.getgroup("collect") + group.addoption( + "--run-optional", + action="append", + dest="run_optional", + default=None, + help="Optional test markers to run; comma-separated", + ) + parser.addini("optional-tests", "List of optional tests markers", "linelist") + + +def pytest_configure(config: "Config") -> None: + """Optional tests are markers. + + Use the syntax in https://docs.pytest.org/en/stable/mark.html#registering-marks. + """ + ot_ini = config.inicfg.get("optional-tests") or [] + ot_markers = set() + ot_run: Set[str] = set() + if isinstance(ot_ini, str): + ot_ini = ot_ini.strip().split("\n") + marker_re = re.compile(r"^\s*(?Pno_)?(?P\w+)(:\s*(?P.*))?") + for ot in ot_ini: + m = marker_re.match(ot) + if not m: + raise ValueError(f"{ot!r} doesn't match pytest marker syntax") + + marker = (m.group("no") or "") + m.group("marker") + description = m.group("description") + config.addinivalue_line("markers", f"{marker}: {description}") + config.addinivalue_line( + "markers", f"{no(marker)}: run when `{marker}` not passed" + ) + ot_markers.add(marker) + + # collect requested optional tests + passed_args = config.getoption("run_optional") + if passed_args: + ot_run.update(itertools.chain.from_iterable(a.split(",") for a in passed_args)) + ot_run |= {no(excluded) for excluded in ot_markers - ot_run} + ot_markers |= {no(m) for m in ot_markers} + + log.info("optional tests to run:", ot_run) + unknown_tests = ot_run - ot_markers + if unknown_tests: + raise ValueError(f"Unknown optional tests wanted: {unknown_tests!r}") + + store = config._store + store[ALL_POSSIBLE_OPTIONAL_MARKERS] = frozenset(ot_markers) + store[ENABLED_OPTIONAL_MARKERS] = frozenset(ot_run) + + +def pytest_collection_modifyitems(config: "Config", items: "List[Node]") -> None: + store = config._store + all_possible_optional_markers = store[ALL_POSSIBLE_OPTIONAL_MARKERS] + enabled_optional_markers = store[ENABLED_OPTIONAL_MARKERS] + + for item in items: + all_markers_on_test = set(m.name for m in item.iter_markers()) + optional_markers_on_test = all_markers_on_test & all_possible_optional_markers + if not optional_markers_on_test or ( + optional_markers_on_test & enabled_optional_markers + ): + continue + log.info("skipping non-requested optional", item) + item.add_marker(skip_mark(frozenset(optional_markers_on_test))) + + +@lru_cache() +def skip_mark(tests: FrozenSet[str]) -> "MarkDecorator": + names = ", ".join(sorted(tests)) + return pytest.mark.skip(reason=f"Marked with disabled optional tests ({names})") + + +@lru_cache() +def no(name: str) -> str: + if name.startswith("no_"): + return name[len("no_") :] + return "no_" + name diff --git a/tests/test_black.py b/tests/test_black.py index 9b2bfcd740e..b8e526a953e 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -460,11 +460,15 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) - @pytest.mark.without_python2 + @pytest.mark.no_python2 def test_python2_should_fail_without_optional_install(self) -> None: - # python 3.7 and below will install typed-ast and will be able to parse Python 2 if sys.version_info < (3, 8): - return + self.skipTest( + "Python 3.6 and 3.7 will install typed-ast to work and as such will be" + " able to parse Python 2 syntax without explicitly specifying the" + " python2 extra" + ) + source = "x = 1234l" tmp_file = Path(black.dump_to_file(source)) try: diff --git a/tox.ini b/tox.ini index cbb0f75d145..317bf485375 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,9 @@ deps = commands = pip install -e .[d] coverage erase - coverage run -m pytest tests -m "not python2" {posargs} + coverage run -m pytest tests --run-optional=no_python2 {posargs} pip install -e .[d,python2] - coverage run -m pytest tests -m "not without_python2" {posargs} + coverage run -m pytest tests --run-optional=python2 {posargs} coverage report [testenv:fuzz] From d0e06b53b09248be34c1d5c0fa8f050bff1d201c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 7 May 2021 16:33:36 +0200 Subject: [PATCH 57/81] Mark blackd tests with the `blackd` optional marker (#2204) This is a follow-up of #2203 that uses a pytest marker instead of a bunch of `skipUnless`. Similarly to the Python 2 tests, they are running by default and will crash on an unsuspecting contributor with missing dependencies. This is by design, we WANT contributors to test everything. Unless we actually don't and then we can run: pytest --run-optional=no_blackd Relatedly, bump required aiohttp to 3.6.0 at least to get rid of expected failures on Python 3.8 (see 6b5eb7d4651c7333cc3f5df4bf7aa7a1f1ffb45b). --- Pipfile | 2 +- pyproject.toml | 1 + setup.py | 2 +- tests/test_blackd.py | 32 +++----------------------------- tests/util.py | 14 +------------- 5 files changed, 7 insertions(+), 44 deletions(-) diff --git a/Pipfile b/Pipfile index 40f3d607653..68562f5e53f 100644 --- a/Pipfile +++ b/Pipfile @@ -20,7 +20,7 @@ wheel = ">=0.31.1" black = {editable = true, extras = ["d"], path = "."} [packages] -aiohttp = ">=3.3.2" +aiohttp = ">=3.6.0" aiohttp-cors = "*" appdirs = "*" click = ">=7.1.2" diff --git a/pyproject.toml b/pyproject.toml index e89cc7a6c9b..4d6777ef6ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,4 +30,5 @@ build-backend = "setuptools.build_meta" # Option below requires `tests/optional.py` optional-tests = [ "no_python2: run when `python2` extra NOT installed", + "no_blackd: run when `d` extra NOT installed", ] \ No newline at end of file diff --git a/setup.py b/setup.py index f1792a46fe8..af93d0f453a 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def get_long_description() -> str: "mypy_extensions>=0.4.3", ], extras_require={ - "d": ["aiohttp>=3.3.2", "aiohttp-cors"], + "d": ["aiohttp>=3.6.0", "aiohttp-cors"], "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], }, diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 9127297c54f..9ca19d49dc6 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -1,10 +1,10 @@ import re -import unittest from unittest.mock import patch from click.testing import CliRunner +import pytest -from tests.util import read_data, DETERMINISTIC_HEADER, skip_if_exception +from tests.util import read_data, DETERMINISTIC_HEADER try: import blackd @@ -16,8 +16,8 @@ has_blackd_deps = True +@pytest.mark.blackd class BlackDTestCase(AioHTTPTestCase): - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") def test_blackd_main(self) -> None: with patch("blackd.web.run_app"): result = CliRunner().invoke(blackd.main, []) @@ -28,10 +28,6 @@ def test_blackd_main(self) -> None: async def get_application(self) -> web.Application: return blackd.make_app() - # TODO: remove these decorators once the below is released - # https://github.com/aio-libs/aiohttp/pull/3727 - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_needs_formatting(self) -> None: response = await self.client.post("/", data=b"print('hello world')") @@ -39,16 +35,12 @@ async def test_blackd_request_needs_formatting(self) -> None: self.assertEqual(response.charset, "utf8") self.assertEqual(await response.read(), b'print("hello world")\n') - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_no_change(self) -> None: response = await self.client.post("/", data=b'print("hello world")\n') self.assertEqual(response.status, 204) self.assertEqual(await response.read(), b"") - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_syntax_error(self) -> None: response = await self.client.post("/", data=b"what even ( is") @@ -59,8 +51,6 @@ async def test_blackd_request_syntax_error(self) -> None: msg=f"Expected error to start with 'Cannot parse', got {repr(content)}", ) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_unsupported_version(self) -> None: response = await self.client.post( @@ -68,8 +58,6 @@ async def test_blackd_unsupported_version(self) -> None: ) self.assertEqual(response.status, 501) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_supported_version(self) -> None: response = await self.client.post( @@ -77,8 +65,6 @@ async def test_blackd_supported_version(self) -> None: ) self.assertEqual(response.status, 200) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_invalid_python_variant(self) -> None: async def check(header_value: str, expected_status: int = 400) -> None: @@ -97,8 +83,6 @@ async def check(header_value: str, expected_status: int = 400) -> None: await check("pypy3.0") await check("jython3.4") - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_pyi(self) -> None: source, expected = read_data("stub.pyi") @@ -108,8 +92,6 @@ async def test_blackd_pyi(self) -> None: self.assertEqual(response.status, 200) self.assertEqual(await response.text(), expected) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_diff(self) -> None: diff_header = re.compile( @@ -128,8 +110,6 @@ async def test_blackd_diff(self) -> None: actual = diff_header.sub(DETERMINISTIC_HEADER, actual) self.assertEqual(actual, expected) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_python_variant(self) -> None: code = ( @@ -166,8 +146,6 @@ async def check(header_value: str, expected_status: int) -> None: await check("py34,py36", 204) await check("34", 204) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_line_length(self) -> None: response = await self.client.post( @@ -175,8 +153,6 @@ async def test_blackd_line_length(self) -> None: ) self.assertEqual(response.status, 200) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_invalid_line_length(self) -> None: response = await self.client.post( @@ -184,8 +160,6 @@ async def test_blackd_invalid_line_length(self) -> None: ) self.assertEqual(response.status, 400) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_response_black_version_header(self) -> None: response = await self.client.post("/") diff --git a/tests/util.py b/tests/util.py index ad9866975ac..3670952ba8c 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,8 +1,7 @@ import os import unittest -from contextlib import contextmanager from pathlib import Path -from typing import List, Tuple, Iterator, Any +from typing import List, Tuple, Any import black from functools import partial @@ -45,17 +44,6 @@ def assertFormatEqual(self, expected: str, actual: str) -> None: self.assertMultiLineEqual(expected, actual) -@contextmanager -def skip_if_exception(e: str) -> Iterator[None]: - try: - yield - except Exception as exc: - if exc.__class__.__name__ == e: - unittest.skip(f"Encountered expected exception {exc}, skipping") - else: - raise - - def read_data(name: str, data: bool = True) -> Tuple[str, str]: """read_data('test_name') -> 'input', 'output'""" if not name.endswith((".py", ".pyi", ".out", ".diff")): From 4fc1354aeb6b217cd18dbdb2a0c41373fa9d8056 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 7 May 2021 10:41:55 -0400 Subject: [PATCH 58/81] Speed up test suite via distributed testing (#2196) * Speed up test suite via distributed testing Since we now run the test suite twice, one with Python 2 and another without, full test runs are getting pretty slow. Let's try to fix that with parallization. Also use verbose mode on CI since more logs is usually better since getting more is quite literally impossible. The main issue we'll face with this is we'll hit https://github.com/pytest-dev/pytest-xdist/issues/620 sometimes (although pretty rarely). I suppose we can test this and see if how bad this bug is for us, and revert if necessary down the line. Also let's have some colours :tada: --- .coveragerc | 1 + .github/workflows/test.yml | 2 +- test_requirements.txt | 2 ++ tox.ini | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index f05041ca1dc..5577e496a57 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ omit = src/blib2to3/* tests/data/* */site-packages/* + .tox/* [run] relative_files = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03adc7f0bea..2cfbab67ce1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: - name: Unit tests run: | - tox -e py + tox -e py -- -v --color=yes - name: Publish coverage to Coveralls # If pushed / is a pull request against main repo AND diff --git a/test_requirements.txt b/test_requirements.txt index a1464e90686..31ab2d05fea 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,5 +3,7 @@ pre-commit pytest >= 6.1.1 pytest-mock >= 3.3.1 pytest-cases >= 2.3.0 +pytest-xdist >= 2.2.1 +pytest-cov >= 2.11.1 parameterized >= 0.7.4 tox diff --git a/tox.ini b/tox.ini index 317bf485375..2379500f55a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,9 @@ deps = commands = pip install -e .[d] coverage erase - coverage run -m pytest tests --run-optional=no_python2 {posargs} + pytest tests --run-optional no_python2 --numprocesses auto --cov {posargs} pip install -e .[d,python2] - coverage run -m pytest tests --run-optional=python2 {posargs} + pytest tests --run-optional python2 --numprocesses auto --cov --cov-append {posargs} coverage report [testenv:fuzz] From f2ea461e9e9fa5c47bb61fd72d512c748928badc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 8 May 2021 11:29:47 +0200 Subject: [PATCH 59/81] Refactor `src/black/__init__.py` into many files (#2206) * Move string-related utility to functions to strings.py, const.py * Move Leaf/Node-related functionality to nodes.py * Move comment-related functions to comments.py * Move caching to cache.py and Mode/TargetVersion/Feature to mode.py * Move some leftover functions to nodes.py, comments.py, strings.py * Add missing files to source list for test runs * Move line-related functionality into lines.py, brackets into brackets.py * Move transformers to trans.py * Move file handling, output, parsing, concurrency, debug, and report * Move two more functions to nodes.py * Add CHANGES * Add numeric.py * Add linegen.py * More docstrings * Include new files in tests Co-authored-by: Jelle Zijlstra --- CHANGES.md | 6 + src/black/__init__.py | 6394 ++------------------------------------ src/black/brackets.py | 334 ++ src/black/cache.py | 83 + src/black/comments.py | 269 ++ src/black/concurrency.py | 39 + src/black/const.py | 4 + src/black/debug.py | 48 + src/black/files.py | 241 ++ src/black/linegen.py | 984 ++++++ src/black/lines.py | 734 +++++ src/black/mode.py | 123 + src/black/nodes.py | 843 +++++ src/black/numerics.py | 65 + src/black/output.py | 83 + src/black/parsing.py | 215 ++ src/black/report.py | 100 + src/black/rusty.py | 28 + src/black/strings.py | 216 ++ src/black/trans.py | 1925 ++++++++++++ tests/test_black.py | 43 +- tests/test_format.py | 30 +- tests/util.py | 19 +- 23 files changed, 6586 insertions(+), 6240 deletions(-) create mode 100644 src/black/brackets.py create mode 100644 src/black/cache.py create mode 100644 src/black/comments.py create mode 100644 src/black/concurrency.py create mode 100644 src/black/const.py create mode 100644 src/black/debug.py create mode 100644 src/black/files.py create mode 100644 src/black/linegen.py create mode 100644 src/black/lines.py create mode 100644 src/black/mode.py create mode 100644 src/black/nodes.py create mode 100644 src/black/numerics.py create mode 100644 src/black/output.py create mode 100644 src/black/parsing.py create mode 100644 src/black/report.py create mode 100644 src/black/rusty.py create mode 100644 src/black/strings.py create mode 100644 src/black/trans.py diff --git a/CHANGES.md b/CHANGES.md index b81e6eb4411..8cbdeca44f7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ ## Change Log +### Unreleased + +#### _Black_ + +- Refactor `src/black/__init__.py` into many files (#2206) + ### 21.5b0 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index e47aa21d435..c61bc8c1d60 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,167 +1,70 @@ -import ast import asyncio -from abc import ABC, abstractmethod -from collections import defaultdict from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor from contextlib import contextmanager from datetime import datetime from enum import Enum -from functools import lru_cache, partial, wraps import io -import itertools -import logging from multiprocessing import Manager, freeze_support import os from pathlib import Path -import pickle import regex as re import signal import sys -import tempfile import tokenize import traceback from typing import ( Any, - Callable, - Collection, Dict, Generator, - Generic, - Iterable, Iterator, List, Optional, Pattern, - Sequence, Set, Sized, Tuple, - Type, - TypeVar, Union, - cast, - TYPE_CHECKING, ) -from mypy_extensions import mypyc_attr -from appdirs import user_cache_dir -from dataclasses import dataclass, field, replace +from dataclasses import replace import click -import toml - -try: - from typed_ast import ast3, ast27 -except ImportError: - if sys.version_info < (3, 8): - print( - "The typed_ast package is not installed.\n" - "You can install it with `python3 -m pip install typed-ast`.", - file=sys.stderr, - ) - sys.exit(1) - else: - ast3 = ast27 = ast -from pathspec import PathSpec +from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES +from black.const import STDIN_PLACEHOLDER +from black.nodes import STARS, syms, is_simple_decorator_expression +from black.lines import Line, EmptyLineTracker +from black.linegen import transform_line, LineGenerator, LN +from black.comments import normalize_fmt_off +from black.mode import Mode, TargetVersion +from black.mode import Feature, supports_feature, VERSION_TO_FEATURES +from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache +from black.concurrency import cancel, shutdown +from black.output import dump_to_file, diff, color_diff, out, err +from black.report import Report, Changed +from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml +from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore +from black.files import wrap_stream_for_windows +from black.parsing import InvalidInput # noqa F401 +from black.parsing import lib2to3_parse, parse_ast, stringify_ast + # lib2to3 fork -from blib2to3.pytree import Node, Leaf, type_repr -from blib2to3 import pygram, pytree -from blib2to3.pgen2 import driver, token -from blib2to3.pgen2.grammar import Grammar -from blib2to3.pgen2.parse import ParseError +from blib2to3.pytree import Node, Leaf +from blib2to3.pgen2 import token from _black_version import version as __version__ -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final - -if TYPE_CHECKING: - import colorama # noqa: F401 - -DEFAULT_LINE_LENGTH = 88 -DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES = r"\.pyi?$" -CACHE_DIR = Path(user_cache_dir("black", version=__version__)) -STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" - -STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. - # types FileContent = str Encoding = str NewLine = str -Depth = int -NodeType = int -ParserState = int -LeafID = int -StringID = int -Priority = int -Index = int -LN = Union[Leaf, Node] -Transformer = Callable[["Line", Collection["Feature"]], Iterator["Line"]] -Timestamp = float -FileSize = int -CacheInfo = Tuple[Timestamp, FileSize] -Cache = Dict[str, CacheInfo] -out = partial(click.secho, bold=True, err=True) -err = partial(click.secho, fg="red", err=True) - -pygram.initialize(CACHE_DIR) -syms = pygram.python_symbols class NothingChanged(UserWarning): """Raised when reformatted code is the same as source.""" -class CannotTransform(Exception): - """Base class for errors raised by Transformers.""" - - -class CannotSplit(CannotTransform): - """A readable split that fits the allotted line length is impossible.""" - - -class InvalidInput(ValueError): - """Raised when input source code fails all parse attempts.""" - - -class BracketMatchError(KeyError): - """Raised when an opening bracket is unable to be matched to a closing bracket.""" - - -T = TypeVar("T") -E = TypeVar("E", bound=Exception) - - -class Ok(Generic[T]): - def __init__(self, value: T) -> None: - self._value = value - - def ok(self) -> T: - return self._value - - -class Err(Generic[E]): - def __init__(self, e: E) -> None: - self._e = e - - def err(self) -> E: - return self._e - - -# The 'Result' return type is used to implement an error-handling model heavily -# influenced by that used by the Rust programming language -# (see https://doc.rust-lang.org/book/ch09-00-error-handling.html). -Result = Union[Ok[T], Err[E]] -TResult = Result[T, CannotTransform] # (T)ransform Result -TMatchResult = TResult[Index] - - class WriteBack(Enum): NO = 0 YES = 1 @@ -182,158 +85,10 @@ def from_configuration( return cls.DIFF if diff else cls.YES -class Changed(Enum): - NO = 0 - CACHED = 1 - YES = 2 - - -class TargetVersion(Enum): - PY27 = 2 - PY33 = 3 - PY34 = 4 - PY35 = 5 - PY36 = 6 - PY37 = 7 - PY38 = 8 - PY39 = 9 - - def is_python2(self) -> bool: - return self is TargetVersion.PY27 - - -class Feature(Enum): - # All string literals are unicode - UNICODE_LITERALS = 1 - F_STRINGS = 2 - NUMERIC_UNDERSCORES = 3 - TRAILING_COMMA_IN_CALL = 4 - TRAILING_COMMA_IN_DEF = 5 - # The following two feature-flags are mutually exclusive, and exactly one should be - # set for every version of python. - ASYNC_IDENTIFIERS = 6 - ASYNC_KEYWORDS = 7 - ASSIGNMENT_EXPRESSIONS = 8 - POS_ONLY_ARGUMENTS = 9 - RELAXED_DECORATORS = 10 - FORCE_OPTIONAL_PARENTHESES = 50 - - -VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { - TargetVersion.PY27: {Feature.ASYNC_IDENTIFIERS}, - TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, - TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, - TargetVersion.PY35: { - Feature.UNICODE_LITERALS, - Feature.TRAILING_COMMA_IN_CALL, - Feature.ASYNC_IDENTIFIERS, - }, - TargetVersion.PY36: { - Feature.UNICODE_LITERALS, - Feature.F_STRINGS, - Feature.NUMERIC_UNDERSCORES, - Feature.TRAILING_COMMA_IN_CALL, - Feature.TRAILING_COMMA_IN_DEF, - Feature.ASYNC_IDENTIFIERS, - }, - TargetVersion.PY37: { - Feature.UNICODE_LITERALS, - Feature.F_STRINGS, - Feature.NUMERIC_UNDERSCORES, - Feature.TRAILING_COMMA_IN_CALL, - Feature.TRAILING_COMMA_IN_DEF, - Feature.ASYNC_KEYWORDS, - }, - TargetVersion.PY38: { - Feature.UNICODE_LITERALS, - Feature.F_STRINGS, - Feature.NUMERIC_UNDERSCORES, - Feature.TRAILING_COMMA_IN_CALL, - Feature.TRAILING_COMMA_IN_DEF, - Feature.ASYNC_KEYWORDS, - Feature.ASSIGNMENT_EXPRESSIONS, - Feature.POS_ONLY_ARGUMENTS, - }, - TargetVersion.PY39: { - Feature.UNICODE_LITERALS, - Feature.F_STRINGS, - Feature.NUMERIC_UNDERSCORES, - Feature.TRAILING_COMMA_IN_CALL, - Feature.TRAILING_COMMA_IN_DEF, - Feature.ASYNC_KEYWORDS, - Feature.ASSIGNMENT_EXPRESSIONS, - Feature.RELAXED_DECORATORS, - Feature.POS_ONLY_ARGUMENTS, - }, -} - - -@dataclass -class Mode: - target_versions: Set[TargetVersion] = field(default_factory=set) - line_length: int = DEFAULT_LINE_LENGTH - string_normalization: bool = True - is_pyi: bool = False - magic_trailing_comma: bool = True - experimental_string_processing: bool = False - - def get_cache_key(self) -> str: - if self.target_versions: - version_str = ",".join( - str(version.value) - for version in sorted(self.target_versions, key=lambda v: v.value) - ) - else: - version_str = "-" - parts = [ - version_str, - str(self.line_length), - str(int(self.string_normalization)), - str(int(self.is_pyi)), - str(int(self.magic_trailing_comma)), - str(int(self.experimental_string_processing)), - ] - return ".".join(parts) - - # Legacy name, left for integrations. FileMode = Mode -def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> bool: - return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) - - -def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: - """Find the absolute filepath to a pyproject.toml if it exists""" - path_project_root = find_project_root(path_search_start) - path_pyproject_toml = path_project_root / "pyproject.toml" - if path_pyproject_toml.is_file(): - return str(path_pyproject_toml) - - try: - path_user_pyproject_toml = find_user_pyproject_toml() - return ( - str(path_user_pyproject_toml) - if path_user_pyproject_toml.is_file() - else None - ) - except PermissionError as e: - # We do not have access to the user-level config directory, so ignore it. - err(f"Ignoring user configuration directory due to {e!r}") - return None - - -def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: - """Parse a pyproject toml file, pulling out relevant parts for Black - - If parsing fails, will raise a toml.TomlDecodeError - """ - pyproject_toml = toml.load(path_config) - config = pyproject_toml.get("tool", {}).get("black", {}) - return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()} - - def read_pyproject_toml( ctx: click.Context, param: click.Parameter, value: Optional[str] ) -> Optional[str]: @@ -349,7 +104,7 @@ def read_pyproject_toml( try: config = parse_pyproject_toml(value) - except (toml.TomlDecodeError, OSError) as e: + except (OSError, ValueError) as e: raise click.FileError( filename=value, hint=f"Error reading configuration file: {e}" ) @@ -391,6 +146,17 @@ def target_version_option_callback( return [TargetVersion[val.upper()] for val in v] +def re_compile_maybe_verbose(regex: str) -> Pattern[str]: + """Compile a regular expression string in `regex`. + + If it contains newlines, use verbose mode. + """ + if "\n" in regex: + regex = "(?x)" + regex + compiled: Pattern[str] = re.compile(regex) + return compiled + + def validate_regex( ctx: click.Context, param: click.Parameter, @@ -945,42 +711,6 @@ def format_file_in_place( return True -def color_diff(contents: str) -> str: - """Inject the ANSI color codes to the diff.""" - lines = contents.split("\n") - for i, line in enumerate(lines): - if line.startswith("+++") or line.startswith("---"): - line = "\033[1;37m" + line + "\033[0m" # bold white, reset - elif line.startswith("@@"): - line = "\033[36m" + line + "\033[0m" # cyan, reset - elif line.startswith("+"): - line = "\033[32m" + line + "\033[0m" # green, reset - elif line.startswith("-"): - line = "\033[31m" + line + "\033[0m" # red, reset - lines[i] = line - return "\n".join(lines) - - -def wrap_stream_for_windows( - f: io.TextIOWrapper, -) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]: - """ - Wrap stream with colorama's wrap_stream so colors are shown on Windows. - - If `colorama` is unavailable, the original stream is returned unmodified. - Otherwise, the `wrap_stream()` function determines whether the stream needs - to be wrapped for a Windows environment and will accordingly either return - an `AnsiToWin32` wrapper or the original stream. - """ - try: - from colorama.initialise import wrap_stream - except ImportError: - return f - else: - # Set `strip=False` to avoid needing to modify test_express_diff_with_color. - return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True) - - def format_stdin_to_stdout( fast: bool, *, write_back: WriteBack = WriteBack.NO, mode: Mode ) -> bool: @@ -1127,5896 +857,197 @@ def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]: return tiow.read(), encoding, newline -def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: - if not target_versions: - # No target_version specified, so try all grammars. - return [ - # Python 3.7+ - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, - # Python 3.0-3.6 - pygram.python_grammar_no_print_statement_no_exec_statement, - # Python 2.7 with future print_function import - pygram.python_grammar_no_print_statement, - # Python 2.7 - pygram.python_grammar, - ] - - if all(version.is_python2() for version in target_versions): - # Python 2-only code, so try Python 2 grammars. - return [ - # Python 2.7 with future print_function import - pygram.python_grammar_no_print_statement, - # Python 2.7 - pygram.python_grammar, - ] - - # Python 3-compatible code, so only try Python 3 grammar. - grammars = [] - # If we have to parse both, try to parse async as a keyword first - if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS): - # Python 3.7+ - grammars.append( - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords - ) - if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): - # Python 3.0-3.6 - grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) - # At least one of the above branches must have been taken, because every Python - # version has exactly one of the two 'ASYNC_*' flags - return grammars - - -def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node: - """Given a string with source, return the lib2to3 Node.""" - if not src_txt.endswith("\n"): - src_txt += "\n" - - for grammar in get_grammars(set(target_versions)): - drv = driver.Driver(grammar, pytree.convert) - try: - result = drv.parse_string(src_txt, True) - break - - except ParseError as pe: - lineno, column = pe.context[1] - lines = src_txt.splitlines() - try: - faulty_line = lines[lineno - 1] - except IndexError: - faulty_line = "" - exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}") - else: - raise exc from None - - if isinstance(result, Leaf): - result = Node(syms.file_input, [result]) - return result - - -def lib2to3_unparse(node: Node) -> str: - """Given a lib2to3 node, return its string representation.""" - code = str(node) - return code - - -class Visitor(Generic[T]): - """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" +def get_features_used(node: Node) -> Set[Feature]: + """Return a set of (relatively) new Python features used in this file. - def visit(self, node: LN) -> Iterator[T]: - """Main method to visit `node` and its children. + Currently looking for: + - f-strings; + - underscores in numeric literals; + - trailing commas after * or ** in function signatures and calls; + - positional only arguments in function signatures and lambdas; + - assignment expression; + - relaxed decorator syntax; + """ + features: Set[Feature] = set() + for n in node.pre_order(): + if n.type == token.STRING: + value_head = n.value[:2] # type: ignore + if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}: + features.add(Feature.F_STRINGS) - It tries to find a `visit_*()` method for the given `node.type`, like - `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. - If no dedicated `visit_*()` method is found, chooses `visit_default()` - instead. + elif n.type == token.NUMBER: + if "_" in n.value: # type: ignore + features.add(Feature.NUMERIC_UNDERSCORES) - Then yields objects of type `T` from the selected visitor. - """ - if node.type < 256: - name = token.tok_name[node.type] - else: - name = str(type_repr(node.type)) - # We explicitly branch on whether a visitor exists (instead of - # using self.visit_default as the default arg to getattr) in order - # to save needing to create a bound method object and so mypyc can - # generate a native call to visit_default. - visitf = getattr(self, f"visit_{name}", None) - if visitf: - yield from visitf(node) - else: - yield from self.visit_default(node) - - def visit_default(self, node: LN) -> Iterator[T]: - """Default `visit_*()` implementation. Recurses to children of `node`.""" - if isinstance(node, Node): - for child in node.children: - yield from self.visit(child) - - -@dataclass -class DebugVisitor(Visitor[T]): - tree_depth: int = 0 - - def visit_default(self, node: LN) -> Iterator[T]: - indent = " " * (2 * self.tree_depth) - if isinstance(node, Node): - _type = type_repr(node.type) - out(f"{indent}{_type}", fg="yellow") - self.tree_depth += 1 - for child in node.children: - yield from self.visit(child) - - self.tree_depth -= 1 - out(f"{indent}/{_type}", fg="yellow", bold=False) - else: - _type = token.tok_name.get(node.type, str(node.type)) - out(f"{indent}{_type}", fg="blue", nl=False) - if node.prefix: - # We don't have to handle prefixes for `Node` objects since - # that delegates to the first child anyway. - out(f" {node.prefix!r}", fg="green", bold=False, nl=False) - out(f" {node.value!r}", fg="blue", bold=False) + elif n.type == token.SLASH: + if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}: + features.add(Feature.POS_ONLY_ARGUMENTS) - @classmethod - def show(cls, code: Union[str, Leaf, Node]) -> None: - """Pretty-print the lib2to3 AST of a given string of `code`. - - Convenience method for debugging. - """ - v: DebugVisitor[None] = DebugVisitor() - if isinstance(code, str): - code = lib2to3_parse(code) - list(v.visit(code)) - - -WHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE} -STATEMENT: Final = { - syms.if_stmt, - syms.while_stmt, - syms.for_stmt, - syms.try_stmt, - syms.except_clause, - syms.with_stmt, - syms.funcdef, - syms.classdef, -} -STANDALONE_COMMENT: Final = 153 -token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT" -LOGIC_OPERATORS: Final = {"and", "or"} -COMPARATORS: Final = { - token.LESS, - token.GREATER, - token.EQEQUAL, - token.NOTEQUAL, - token.LESSEQUAL, - token.GREATEREQUAL, -} -MATH_OPERATORS: Final = { - token.VBAR, - token.CIRCUMFLEX, - token.AMPER, - token.LEFTSHIFT, - token.RIGHTSHIFT, - token.PLUS, - token.MINUS, - token.STAR, - token.SLASH, - token.DOUBLESLASH, - token.PERCENT, - token.AT, - token.TILDE, - token.DOUBLESTAR, -} -STARS: Final = {token.STAR, token.DOUBLESTAR} -VARARGS_SPECIALS: Final = STARS | {token.SLASH} -VARARGS_PARENTS: Final = { - syms.arglist, - syms.argument, # double star in arglist - syms.trailer, # single argument to call - syms.typedargslist, - syms.varargslist, # lambdas -} -UNPACKING_PARENTS: Final = { - syms.atom, # single element of a list or set literal - syms.dictsetmaker, - syms.listmaker, - syms.testlist_gexp, - syms.testlist_star_expr, -} -TEST_DESCENDANTS: Final = { - syms.test, - syms.lambdef, - syms.or_test, - syms.and_test, - syms.not_test, - syms.comparison, - syms.star_expr, - syms.expr, - syms.xor_expr, - syms.and_expr, - syms.shift_expr, - syms.arith_expr, - syms.trailer, - syms.term, - syms.power, -} -ASSIGNMENTS: Final = { - "=", - "+=", - "-=", - "*=", - "@=", - "/=", - "%=", - "&=", - "|=", - "^=", - "<<=", - ">>=", - "**=", - "//=", -} -COMPREHENSION_PRIORITY: Final = 20 -COMMA_PRIORITY: Final = 18 -TERNARY_PRIORITY: Final = 16 -LOGIC_PRIORITY: Final = 14 -STRING_PRIORITY: Final = 12 -COMPARATOR_PRIORITY: Final = 10 -MATH_PRIORITIES: Final = { - token.VBAR: 9, - token.CIRCUMFLEX: 8, - token.AMPER: 7, - token.LEFTSHIFT: 6, - token.RIGHTSHIFT: 6, - token.PLUS: 5, - token.MINUS: 5, - token.STAR: 4, - token.SLASH: 4, - token.DOUBLESLASH: 4, - token.PERCENT: 4, - token.AT: 4, - token.TILDE: 3, - token.DOUBLESTAR: 2, -} -DOT_PRIORITY: Final = 1 - - -@dataclass -class BracketTracker: - """Keeps track of brackets on a line.""" - - depth: int = 0 - bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = field(default_factory=dict) - delimiters: Dict[LeafID, Priority] = field(default_factory=dict) - previous: Optional[Leaf] = None - _for_loop_depths: List[int] = field(default_factory=list) - _lambda_argument_depths: List[int] = field(default_factory=list) - invisible: List[Leaf] = field(default_factory=list) - - def mark(self, leaf: Leaf) -> None: - """Mark `leaf` with bracket-related metadata. Keep track of delimiters. - - All leaves receive an int `bracket_depth` field that stores how deep - within brackets a given leaf is. 0 means there are no enclosing brackets - that started on this line. - - If a leaf is itself a closing bracket, it receives an `opening_bracket` - field that it forms a pair with. This is a one-directional link to - avoid reference cycles. - - If a leaf is a delimiter (a token on which Black can split the line if - needed) and it's on depth 0, its `id()` is stored in the tracker's - `delimiters` field. - """ - if leaf.type == token.COMMENT: - return - - self.maybe_decrement_after_for_loop_variable(leaf) - self.maybe_decrement_after_lambda_arguments(leaf) - if leaf.type in CLOSING_BRACKETS: - self.depth -= 1 - try: - opening_bracket = self.bracket_match.pop((self.depth, leaf.type)) - except KeyError as e: - raise BracketMatchError( - "Unable to match a closing bracket to the following opening" - f" bracket: {leaf}" - ) from e - leaf.opening_bracket = opening_bracket - if not leaf.value: - self.invisible.append(leaf) - leaf.bracket_depth = self.depth - if self.depth == 0: - delim = is_split_before_delimiter(leaf, self.previous) - if delim and self.previous is not None: - self.delimiters[id(self.previous)] = delim - else: - delim = is_split_after_delimiter(leaf, self.previous) - if delim: - self.delimiters[id(leaf)] = delim - if leaf.type in OPENING_BRACKETS: - self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf - self.depth += 1 - if not leaf.value: - self.invisible.append(leaf) - self.previous = leaf - self.maybe_increment_lambda_arguments(leaf) - self.maybe_increment_for_loop_variable(leaf) - - def any_open_brackets(self) -> bool: - """Return True if there is an yet unmatched open bracket on the line.""" - return bool(self.bracket_match) - - def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> Priority: - """Return the highest priority of a delimiter found on the line. - - Values are consistent with what `is_split_*_delimiter()` return. - Raises ValueError on no delimiters. - """ - return max(v for k, v in self.delimiters.items() if k not in exclude) - - def delimiter_count_with_priority(self, priority: Priority = 0) -> int: - """Return the number of delimiters with the given `priority`. - - If no `priority` is passed, defaults to max priority on the line. - """ - if not self.delimiters: - return 0 - - priority = priority or self.max_delimiter_priority() - return sum(1 for p in self.delimiters.values() if p == priority) - - def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: - """In a for loop, or comprehension, the variables are often unpacks. - - To avoid splitting on the comma in this situation, increase the depth of - tokens between `for` and `in`. - """ - if leaf.type == token.NAME and leaf.value == "for": - self.depth += 1 - self._for_loop_depths.append(self.depth) - return True + elif n.type == token.COLONEQUAL: + features.add(Feature.ASSIGNMENT_EXPRESSIONS) - return False + elif n.type == syms.decorator: + if len(n.children) > 1 and not is_simple_decorator_expression( + n.children[1] + ): + features.add(Feature.RELAXED_DECORATORS) - def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool: - """See `maybe_increment_for_loop_variable` above for explanation.""" - if ( - self._for_loop_depths - and self._for_loop_depths[-1] == self.depth - and leaf.type == token.NAME - and leaf.value == "in" + elif ( + n.type in {syms.typedargslist, syms.arglist} + and n.children + and n.children[-1].type == token.COMMA ): - self.depth -= 1 - self._for_loop_depths.pop() - return True - - return False + if n.type == syms.typedargslist: + feature = Feature.TRAILING_COMMA_IN_DEF + else: + feature = Feature.TRAILING_COMMA_IN_CALL - def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool: - """In a lambda expression, there might be more than one argument. + for ch in n.children: + if ch.type in STARS: + features.add(feature) - To avoid splitting on the comma in this situation, increase the depth of - tokens between `lambda` and `:`. - """ - if leaf.type == token.NAME and leaf.value == "lambda": - self.depth += 1 - self._lambda_argument_depths.append(self.depth) - return True + if ch.type == syms.argument: + for argch in ch.children: + if argch.type in STARS: + features.add(feature) - return False + return features - def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool: - """See `maybe_increment_lambda_arguments` above for explanation.""" - if ( - self._lambda_argument_depths - and self._lambda_argument_depths[-1] == self.depth - and leaf.type == token.COLON - ): - self.depth -= 1 - self._lambda_argument_depths.pop() - return True - return False +def detect_target_versions(node: Node) -> Set[TargetVersion]: + """Detect the version to target based on the nodes used.""" + features = get_features_used(node) + return { + version for version in TargetVersion if features <= VERSION_TO_FEATURES[version] + } - def get_open_lsqb(self) -> Optional[Leaf]: - """Return the most recent opening square bracket (if any).""" - return self.bracket_match.get((self.depth - 1, token.RSQB)) - - -@dataclass -class Line: - """Holds leaves and comments. Can be printed with `str(line)`.""" - - mode: Mode - depth: int = 0 - leaves: List[Leaf] = field(default_factory=list) - # keys ordered like `leaves` - comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) - bracket_tracker: BracketTracker = field(default_factory=BracketTracker) - inside_brackets: bool = False - should_split_rhs: bool = False - magic_trailing_comma: Optional[Leaf] = None - - def append(self, leaf: Leaf, preformatted: bool = False) -> None: - """Add a new `leaf` to the end of the line. - - Unless `preformatted` is True, the `leaf` will receive a new consistent - whitespace prefix and metadata applied by :class:`BracketTracker`. - Trailing commas are maybe removed, unpacked for loop variables are - demoted from being delimiters. - - Inline comments are put aside. - """ - has_value = leaf.type in BRACKETS or bool(leaf.value.strip()) - if not has_value: - return - - if token.COLON == leaf.type and self.is_class_paren_empty: - del self.leaves[-2:] - if self.leaves and not preformatted: - # Note: at this point leaf.prefix should be empty except for - # imports, for which we only preserve newlines. - leaf.prefix += whitespace( - leaf, complex_subscript=self.is_complex_subscript(leaf) - ) - if self.inside_brackets or not preformatted: - self.bracket_tracker.mark(leaf) - if self.mode.magic_trailing_comma: - if self.has_magic_trailing_comma(leaf): - self.magic_trailing_comma = leaf - elif self.has_magic_trailing_comma(leaf, ensure_removable=True): - self.remove_trailing_comma() - if not self.append_comment(leaf): - self.leaves.append(leaf) - - def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: - """Like :func:`append()` but disallow invalid standalone comment structure. - - Raises ValueError when any `leaf` is appended after a standalone comment - or when a standalone comment is not the first leaf on the line. - """ - if self.bracket_tracker.depth == 0: - if self.is_comment: - raise ValueError("cannot append to standalone comments") - - if self.leaves and leaf.type == STANDALONE_COMMENT: - raise ValueError( - "cannot append standalone comments to a populated line" - ) - self.append(leaf, preformatted=preformatted) - - @property - def is_comment(self) -> bool: - """Is this line a standalone comment?""" - return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT - - @property - def is_decorator(self) -> bool: - """Is this line a decorator?""" - return bool(self) and self.leaves[0].type == token.AT - - @property - def is_import(self) -> bool: - """Is this an import line?""" - return bool(self) and is_import(self.leaves[0]) - - @property - def is_class(self) -> bool: - """Is this line a class definition?""" - return ( - bool(self) - and self.leaves[0].type == token.NAME - and self.leaves[0].value == "class" - ) +def get_future_imports(node: Node) -> Set[str]: + """Return a set of __future__ imports in the file.""" + imports: Set[str] = set() - @property - def is_stub_class(self) -> bool: - """Is this line a class definition with a body consisting only of "..."?""" - return self.is_class and self.leaves[-3:] == [ - Leaf(token.DOT, ".") for _ in range(3) - ] - - @property - def is_def(self) -> bool: - """Is this a function definition? (Also returns True for async defs.)""" - try: - first_leaf = self.leaves[0] - except IndexError: - return False - - try: - second_leaf: Optional[Leaf] = self.leaves[1] - except IndexError: - second_leaf = None - return (first_leaf.type == token.NAME and first_leaf.value == "def") or ( - first_leaf.type == token.ASYNC - and second_leaf is not None - and second_leaf.type == token.NAME - and second_leaf.value == "def" - ) + def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]: + for child in children: + if isinstance(child, Leaf): + if child.type == token.NAME: + yield child.value - @property - def is_class_paren_empty(self) -> bool: - """Is this a class with no base classes but using parentheses? - - Those are unnecessary and should be removed. - """ - return ( - bool(self) - and len(self.leaves) == 4 - and self.is_class - and self.leaves[2].type == token.LPAR - and self.leaves[2].value == "(" - and self.leaves[3].type == token.RPAR - and self.leaves[3].value == ")" - ) + elif child.type == syms.import_as_name: + orig_name = child.children[0] + assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports" + assert orig_name.type == token.NAME, "Invalid syntax parsing imports" + yield orig_name.value - @property - def is_triple_quoted_string(self) -> bool: - """Is the line a triple quoted string?""" - return ( - bool(self) - and self.leaves[0].type == token.STRING - and self.leaves[0].value.startswith(('"""', "'''")) - ) + elif child.type == syms.import_as_names: + yield from get_imports_from_children(child.children) - def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: - """If so, needs to be split before emitting.""" - for leaf in self.leaves: - if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit: - return True + else: + raise AssertionError("Invalid syntax parsing imports") - return False + for child in node.children: + if child.type != syms.simple_stmt: + break - def contains_uncollapsable_type_comments(self) -> bool: - ignored_ids = set() - try: - last_leaf = self.leaves[-1] - ignored_ids.add(id(last_leaf)) - if last_leaf.type == token.COMMA or ( - last_leaf.type == token.RPAR and not last_leaf.value + first_child = child.children[0] + if isinstance(first_child, Leaf): + # Continue looking if we see a docstring; otherwise stop. + if ( + len(child.children) == 2 + and first_child.type == token.STRING + and child.children[1].type == token.NEWLINE ): - # When trailing commas or optional parens are inserted by Black for - # consistency, comments after the previous last element are not moved - # (they don't have to, rendering will still be correct). So we ignore - # trailing commas and invisible. - last_leaf = self.leaves[-2] - ignored_ids.add(id(last_leaf)) - except IndexError: - return False - - # A type comment is uncollapsable if it is attached to a leaf - # that isn't at the end of the line (since that could cause it - # to get associated to a different argument) or if there are - # comments before it (since that could cause it to get hidden - # behind a comment. - comment_seen = False - for leaf_id, comments in self.comments.items(): - for comment in comments: - if is_type_comment(comment): - if comment_seen or ( - not is_type_comment(comment, " ignore") - and leaf_id not in ignored_ids - ): - return True - - comment_seen = True - - return False - - def contains_unsplittable_type_ignore(self) -> bool: - if not self.leaves: - return False - - # If a 'type: ignore' is attached to the end of a line, we - # can't split the line, because we can't know which of the - # subexpressions the ignore was meant to apply to. - # - # We only want this to apply to actual physical lines from the - # original source, though: we don't want the presence of a - # 'type: ignore' at the end of a multiline expression to - # justify pushing it all onto one line. Thus we - # (unfortunately) need to check the actual source lines and - # only report an unsplittable 'type: ignore' if this line was - # one line in the original code. - - # Grab the first and last line numbers, skipping generated leaves - first_line = next((leaf.lineno for leaf in self.leaves if leaf.lineno != 0), 0) - last_line = next( - (leaf.lineno for leaf in reversed(self.leaves) if leaf.lineno != 0), 0 - ) - - if first_line == last_line: - # We look at the last two leaves since a comma or an - # invisible paren could have been added at the end of the - # line. - for node in self.leaves[-2:]: - for comment in self.comments.get(id(node), []): - if is_type_comment(comment, " ignore"): - return True - - return False - - def contains_multiline_strings(self) -> bool: - return any(is_multiline_string(leaf) for leaf in self.leaves) - - def has_magic_trailing_comma( - self, closing: Leaf, ensure_removable: bool = False - ) -> bool: - """Return True if we have a magic trailing comma, that is when: - - there's a trailing comma here - - it's not a one-tuple - Additionally, if ensure_removable: - - it's not from square bracket indexing - """ - if not ( - closing.type in CLOSING_BRACKETS - and self.leaves - and self.leaves[-1].type == token.COMMA - ): - return False - - if closing.type == token.RBRACE: - return True - - if closing.type == token.RSQB: - if not ensure_removable: - return True - comma = self.leaves[-1] - return bool(comma.parent and comma.parent.type == syms.listmaker) + continue - if self.is_import: - return True + break - if not is_one_tuple_between(closing.opening_bracket, closing, self.leaves): - return True + elif first_child.type == syms.import_from: + module_name = first_child.children[1] + if not isinstance(module_name, Leaf) or module_name.value != "__future__": + break - return False + imports |= set(get_imports_from_children(first_child.children[3:])) + else: + break - def append_comment(self, comment: Leaf) -> bool: - """Add an inline or standalone comment to the line.""" - if ( - comment.type == STANDALONE_COMMENT - and self.bracket_tracker.any_open_brackets() - ): - comment.prefix = "" - return False - - if comment.type != token.COMMENT: - return False - - if not self.leaves: - comment.type = STANDALONE_COMMENT - comment.prefix = "" - return False - - last_leaf = self.leaves[-1] - if ( - last_leaf.type == token.RPAR - and not last_leaf.value - and last_leaf.parent - and len(list(last_leaf.parent.leaves())) <= 3 - and not is_type_comment(comment) - ): - # Comments on an optional parens wrapping a single leaf should belong to - # the wrapped node except if it's a type comment. Pinning the comment like - # this avoids unstable formatting caused by comment migration. - if len(self.leaves) < 2: - comment.type = STANDALONE_COMMENT - comment.prefix = "" - return False - - last_leaf = self.leaves[-2] - self.comments.setdefault(id(last_leaf), []).append(comment) - return True + return imports - def comments_after(self, leaf: Leaf) -> List[Leaf]: - """Generate comments that should appear directly after `leaf`.""" - return self.comments.get(id(leaf), []) - def remove_trailing_comma(self) -> None: - """Remove the trailing comma and moves the comments attached to it.""" - trailing_comma = self.leaves.pop() - trailing_comma_comments = self.comments.pop(id(trailing_comma), []) - self.comments.setdefault(id(self.leaves[-1]), []).extend( - trailing_comma_comments +def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None: + """Raise AssertionError if `src` and `dst` aren't equivalent.""" + try: + src_ast = parse_ast(src) + except Exception as exc: + raise AssertionError( + "cannot use --safe with this file; failed to parse source file. AST" + f" error message: {exc}" ) - def is_complex_subscript(self, leaf: Leaf) -> bool: - """Return True iff `leaf` is part of a slice with non-trivial exprs.""" - open_lsqb = self.bracket_tracker.get_open_lsqb() - if open_lsqb is None: - return False - - subscript_start = open_lsqb.next_sibling + try: + dst_ast = parse_ast(dst) + except Exception as exc: + log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst) + raise AssertionError( + f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. " + "Please report a bug on https://github.com/psf/black/issues. " + f"This invalid output might be helpful: {log}" + ) from None - if isinstance(subscript_start, Node): - if subscript_start.type == syms.listmaker: - return False + src_ast_str = "\n".join(stringify_ast(src_ast)) + dst_ast_str = "\n".join(stringify_ast(dst_ast)) + if src_ast_str != dst_ast_str: + log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst")) + raise AssertionError( + "INTERNAL ERROR: Black produced code that is not equivalent to the" + f" source on pass {pass_num}. Please report a bug on " + f"https://github.com/psf/black/issues. This diff might be helpful: {log}" + ) from None - if subscript_start.type == syms.subscriptlist: - subscript_start = child_towards(subscript_start, leaf) - return subscript_start is not None and any( - n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() - ) - def clone(self) -> "Line": - return Line( - mode=self.mode, - depth=self.depth, - inside_brackets=self.inside_brackets, - should_split_rhs=self.should_split_rhs, - magic_trailing_comma=self.magic_trailing_comma, +def assert_stable(src: str, dst: str, mode: Mode) -> None: + """Raise AssertionError if `dst` reformats differently the second time.""" + newdst = format_str(dst, mode=mode) + if dst != newdst: + log = dump_to_file( + str(mode), + diff(src, dst, "source", "first pass"), + diff(dst, newdst, "first pass", "second pass"), ) + raise AssertionError( + "INTERNAL ERROR: Black produced different code on the second pass of the" + " formatter. Please report a bug on https://github.com/psf/black/issues." + f" This diff might be helpful: {log}" + ) from None - def __str__(self) -> str: - """Render the line.""" - if not self: - return "\n" - indent = " " * self.depth - leaves = iter(self.leaves) - first = next(leaves) - res = f"{first.prefix}{indent}{first.value}" - for leaf in leaves: - res += str(leaf) - for comment in itertools.chain.from_iterable(self.comments.values()): - res += str(comment) +@contextmanager +def nullcontext() -> Iterator[None]: + """Return an empty context manager. - return res + "\n" + To be used like `nullcontext` in Python 3.7. + """ + yield - def __bool__(self) -> bool: - """Return True if the line has leaves or comments.""" - return bool(self.leaves or self.comments) +def patch_click() -> None: + """Make Click not crash. -@dataclass -class EmptyLineTracker: - """Provides a stateful method that returns the number of potential extra - empty lines needed before and after the currently processed line. + On certain misconfigured environments, Python 3 selects the ASCII encoding as the + default which restricts paths that it can access during the lifetime of the + application. Click refuses to work in this scenario by raising a RuntimeError. - Note: this tracker works on lines that haven't been split yet. It assumes - the prefix of the first leaf consists of optional newlines. Those newlines - are consumed by `maybe_empty_lines()` and included in the computation. + In case of Black the likelihood that non-ASCII characters are going to be used in + file paths is minimal since it's Python source code. Moreover, this crash was + spurious on Python 3.7 thanks to PEP 538 and PEP 540. """ + try: + from click import core + from click import _unicodefun # type: ignore + except ModuleNotFoundError: + return - is_pyi: bool = False - previous_line: Optional[Line] = None - previous_after: int = 0 - previous_defs: List[int] = field(default_factory=list) - - def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: - """Return the number of extra empty lines before and after the `current_line`. - - This is for separating `def`, `async def` and `class` with extra empty - lines (two on module-level). - """ - before, after = self._maybe_empty_lines(current_line) - before = ( - # Black should not insert empty lines at the beginning - # of the file - 0 - if self.previous_line is None - else before - self.previous_after - ) - self.previous_after = after - self.previous_line = current_line - return before, after - - def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: - max_allowed = 1 - if current_line.depth == 0: - max_allowed = 1 if self.is_pyi else 2 - if current_line.leaves: - # Consume the first leaf's extra newlines. - first_leaf = current_line.leaves[0] - before = first_leaf.prefix.count("\n") - before = min(before, max_allowed) - first_leaf.prefix = "" - else: - before = 0 - depth = current_line.depth - while self.previous_defs and self.previous_defs[-1] >= depth: - self.previous_defs.pop() - if self.is_pyi: - before = 0 if depth else 1 - else: - before = 1 if depth else 2 - if current_line.is_decorator or current_line.is_def or current_line.is_class: - return self._maybe_empty_lines_for_class_or_def(current_line, before) - - if ( - self.previous_line - and self.previous_line.is_import - and not current_line.is_import - and depth == self.previous_line.depth - ): - return (before or 1), 0 - - if ( - self.previous_line - and self.previous_line.is_class - and current_line.is_triple_quoted_string - ): - return before, 1 - - return before, 0 - - def _maybe_empty_lines_for_class_or_def( - self, current_line: Line, before: int - ) -> Tuple[int, int]: - if not current_line.is_decorator: - self.previous_defs.append(current_line.depth) - if self.previous_line is None: - # Don't insert empty lines before the first line in the file. - return 0, 0 - - if self.previous_line.is_decorator: - if self.is_pyi and current_line.is_stub_class: - # Insert an empty line after a decorated stub class - return 0, 1 - - return 0, 0 - - if self.previous_line.depth < current_line.depth and ( - self.previous_line.is_class or self.previous_line.is_def - ): - return 0, 0 - - if ( - self.previous_line.is_comment - and self.previous_line.depth == current_line.depth - and before == 0 - ): - return 0, 0 - - if self.is_pyi: - if self.previous_line.depth > current_line.depth: - newlines = 1 - elif current_line.is_class or self.previous_line.is_class: - if current_line.is_stub_class and self.previous_line.is_stub_class: - # No blank line between classes with an empty body - newlines = 0 - else: - newlines = 1 - elif ( - current_line.is_def or current_line.is_decorator - ) and not self.previous_line.is_def: - # Blank line between a block of functions (maybe with preceding - # decorators) and a block of non-functions - newlines = 1 - else: - newlines = 0 - else: - newlines = 2 - if current_line.depth and newlines: - newlines -= 1 - return newlines, 0 - - -@dataclass -class LineGenerator(Visitor[Line]): - """Generates reformatted Line objects. Empty lines are not emitted. - - Note: destroys the tree it's visiting by mutating prefixes of its leaves - in ways that will no longer stringify to valid Python code on the tree. - """ - - mode: Mode - remove_u_prefix: bool = False - current_line: Line = field(init=False) - - def line(self, indent: int = 0) -> Iterator[Line]: - """Generate a line. - - If the line is empty, only emit if it makes sense. - If the line is too long, split it first and then generate. - - If any lines were generated, set up a new current_line. - """ - if not self.current_line: - self.current_line.depth += indent - return # Line is empty, don't emit. Creating a new one unnecessary. - - complete_line = self.current_line - self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent) - yield complete_line - - def visit_default(self, node: LN) -> Iterator[Line]: - """Default `visit_*()` implementation. Recurses to children of `node`.""" - if isinstance(node, Leaf): - any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() - for comment in generate_comments(node): - if any_open_brackets: - # any comment within brackets is subject to splitting - self.current_line.append(comment) - elif comment.type == token.COMMENT: - # regular trailing comment - self.current_line.append(comment) - yield from self.line() - - else: - # regular standalone comment - yield from self.line() - - self.current_line.append(comment) - yield from self.line() - - normalize_prefix(node, inside_brackets=any_open_brackets) - if self.mode.string_normalization and node.type == token.STRING: - normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix) - normalize_string_quotes(node) - if node.type == token.NUMBER: - normalize_numeric_literal(node) - if node.type not in WHITESPACE: - self.current_line.append(node) - yield from super().visit_default(node) - - def visit_INDENT(self, node: Leaf) -> Iterator[Line]: - """Increase indentation level, maybe yield a line.""" - # In blib2to3 INDENT never holds comments. - yield from self.line(+1) - yield from self.visit_default(node) - - def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: - """Decrease indentation level, maybe yield a line.""" - # The current line might still wait for trailing comments. At DEDENT time - # there won't be any (they would be prefixes on the preceding NEWLINE). - # Emit the line then. - yield from self.line() - - # While DEDENT has no value, its prefix may contain standalone comments - # that belong to the current indentation level. Get 'em. - yield from self.visit_default(node) - - # Finally, emit the dedent. - yield from self.line(-1) - - def visit_stmt( - self, node: Node, keywords: Set[str], parens: Set[str] - ) -> Iterator[Line]: - """Visit a statement. - - This implementation is shared for `if`, `while`, `for`, `try`, `except`, - `def`, `with`, `class`, `assert` and assignments. - - The relevant Python language `keywords` for a given statement will be - NAME leaves within it. This methods puts those on a separate line. - - `parens` holds a set of string leaf values immediately after which - invisible parens should be put. - """ - normalize_invisible_parens(node, parens_after=parens) - for child in node.children: - if child.type == token.NAME and child.value in keywords: # type: ignore - yield from self.line() - - yield from self.visit(child) - - def visit_suite(self, node: Node) -> Iterator[Line]: - """Visit a suite.""" - if self.mode.is_pyi and is_stub_suite(node): - yield from self.visit(node.children[2]) - else: - yield from self.visit_default(node) - - def visit_simple_stmt(self, node: Node) -> Iterator[Line]: - """Visit a statement without nested statements.""" - if first_child_is_arith(node): - wrap_in_parentheses(node, node.children[0], visible=False) - is_suite_like = node.parent and node.parent.type in STATEMENT - if is_suite_like: - if self.mode.is_pyi and is_stub_body(node): - yield from self.visit_default(node) - else: - yield from self.line(+1) - yield from self.visit_default(node) - yield from self.line(-1) - - else: - if ( - not self.mode.is_pyi - or not node.parent - or not is_stub_suite(node.parent) - ): - yield from self.line() - yield from self.visit_default(node) - - def visit_async_stmt(self, node: Node) -> Iterator[Line]: - """Visit `async def`, `async for`, `async with`.""" - yield from self.line() - - children = iter(node.children) - for child in children: - yield from self.visit(child) - - if child.type == token.ASYNC: - break - - internal_stmt = next(children) - for child in internal_stmt.children: - yield from self.visit(child) - - def visit_decorators(self, node: Node) -> Iterator[Line]: - """Visit decorators.""" - for child in node.children: - yield from self.line() - yield from self.visit(child) - - def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: - """Remove a semicolon and put the other statement on a separate line.""" - yield from self.line() - - def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: - """End of file. Process outstanding comments and end with a newline.""" - yield from self.visit_default(leaf) - yield from self.line() - - def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: - if not self.current_line.bracket_tracker.any_open_brackets(): - yield from self.line() - yield from self.visit_default(leaf) - - def visit_factor(self, node: Node) -> Iterator[Line]: - """Force parentheses between a unary op and a binary power: - - -2 ** 8 -> -(2 ** 8) - """ - _operator, operand = node.children - if ( - operand.type == syms.power - and len(operand.children) == 3 - and operand.children[1].type == token.DOUBLESTAR - ): - lpar = Leaf(token.LPAR, "(") - rpar = Leaf(token.RPAR, ")") - index = operand.remove() or 0 - node.insert_child(index, Node(syms.atom, [lpar, operand, rpar])) - yield from self.visit_default(node) - - def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: - if is_docstring(leaf) and "\\\n" not in leaf.value: - # We're ignoring docstrings with backslash newline escapes because changing - # indentation of those changes the AST representation of the code. - prefix = get_string_prefix(leaf.value) - docstring = leaf.value[len(prefix) :] # Remove the prefix - quote_char = docstring[0] - # A natural way to remove the outer quotes is to do: - # docstring = docstring.strip(quote_char) - # but that breaks on """""x""" (which is '""x'). - # So we actually need to remove the first character and the next two - # characters but only if they are the same as the first. - quote_len = 1 if docstring[1] != quote_char else 3 - docstring = docstring[quote_len:-quote_len] - - if is_multiline_string(leaf): - indent = " " * 4 * self.current_line.depth - docstring = fix_docstring(docstring, indent) - else: - docstring = docstring.strip() - - if docstring: - # Add some padding if the docstring starts / ends with a quote mark. - if docstring[0] == quote_char: - docstring = " " + docstring - if docstring[-1] == quote_char: - docstring += " " - if docstring[-1] == "\\": - backslash_count = len(docstring) - len(docstring.rstrip("\\")) - if backslash_count % 2: - # Odd number of tailing backslashes, add some padding to - # avoid escaping the closing string quote. - docstring += " " - else: - # Add some padding if the docstring is empty. - docstring = " " - - # We could enforce triple quotes at this point. - quote = quote_char * quote_len - leaf.value = prefix + quote + docstring + quote - - yield from self.visit_default(leaf) - - def __post_init__(self) -> None: - """You are in a twisty little maze of passages.""" - self.current_line = Line(mode=self.mode) - - v = self.visit_stmt - Ø: Set[str] = set() - self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","}) - self.visit_if_stmt = partial( - v, keywords={"if", "else", "elif"}, parens={"if", "elif"} - ) - self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"}) - self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"}) - self.visit_try_stmt = partial( - v, keywords={"try", "except", "else", "finally"}, parens=Ø - ) - self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø) - self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø) - self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø) - self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) - self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) - self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) - self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) - self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) - self.visit_async_funcdef = self.visit_async_stmt - self.visit_decorated = self.visit_decorators - - -IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist} -BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE} -OPENING_BRACKETS = set(BRACKET.keys()) -CLOSING_BRACKETS = set(BRACKET.values()) -BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS -ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT} - - -def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 - """Return whitespace prefix if needed for the given `leaf`. - - `complex_subscript` signals whether the given leaf is part of a subscription - which has non-trivial arguments, like arithmetic expressions or function calls. - """ - NO = "" - SPACE = " " - DOUBLESPACE = " " - t = leaf.type - p = leaf.parent - v = leaf.value - if t in ALWAYS_NO_SPACE: - return NO - - if t == token.COMMENT: - return DOUBLESPACE - - assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}" - if t == token.COLON and p.type not in { - syms.subscript, - syms.subscriptlist, - syms.sliceop, - }: - return NO - - prev = leaf.prev_sibling - if not prev: - prevp = preceding_leaf(p) - if not prevp or prevp.type in OPENING_BRACKETS: - return NO - - if t == token.COLON: - if prevp.type == token.COLON: - return NO - - elif prevp.type != token.COMMA and not complex_subscript: - return NO - - return SPACE - - if prevp.type == token.EQUAL: - if prevp.parent: - if prevp.parent.type in { - syms.arglist, - syms.argument, - syms.parameters, - syms.varargslist, - }: - return NO - - elif prevp.parent.type == syms.typedargslist: - # A bit hacky: if the equal sign has whitespace, it means we - # previously found it's a typed argument. So, we're using - # that, too. - return prevp.prefix - - elif prevp.type in VARARGS_SPECIALS: - if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS): - return NO - - elif prevp.type == token.COLON: - if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}: - return SPACE if complex_subscript else NO - - elif ( - prevp.parent - and prevp.parent.type == syms.factor - and prevp.type in MATH_OPERATORS - ): - return NO - - elif ( - prevp.type == token.RIGHTSHIFT - and prevp.parent - and prevp.parent.type == syms.shift_expr - and prevp.prev_sibling - and prevp.prev_sibling.type == token.NAME - and prevp.prev_sibling.value == "print" # type: ignore - ): - # Python 2 print chevron - return NO - elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator: - # no space in decorators - return NO - - elif prev.type in OPENING_BRACKETS: - return NO - - if p.type in {syms.parameters, syms.arglist}: - # untyped function signatures or calls - if not prev or prev.type != token.COMMA: - return NO - - elif p.type == syms.varargslist: - # lambdas - if prev and prev.type != token.COMMA: - return NO - - elif p.type == syms.typedargslist: - # typed function signatures - if not prev: - return NO - - if t == token.EQUAL: - if prev.type != syms.tname: - return NO - - elif prev.type == token.EQUAL: - # A bit hacky: if the equal sign has whitespace, it means we - # previously found it's a typed argument. So, we're using that, too. - return prev.prefix - - elif prev.type != token.COMMA: - return NO - - elif p.type == syms.tname: - # type names - if not prev: - prevp = preceding_leaf(p) - if not prevp or prevp.type != token.COMMA: - return NO - - elif p.type == syms.trailer: - # attributes and calls - if t == token.LPAR or t == token.RPAR: - return NO - - if not prev: - if t == token.DOT: - prevp = preceding_leaf(p) - if not prevp or prevp.type != token.NUMBER: - return NO - - elif t == token.LSQB: - return NO - - elif prev.type != token.COMMA: - return NO - - elif p.type == syms.argument: - # single argument - if t == token.EQUAL: - return NO - - if not prev: - prevp = preceding_leaf(p) - if not prevp or prevp.type == token.LPAR: - return NO - - elif prev.type in {token.EQUAL} | VARARGS_SPECIALS: - return NO - - elif p.type == syms.decorator: - # decorators - return NO - - elif p.type == syms.dotted_name: - if prev: - return NO - - prevp = preceding_leaf(p) - if not prevp or prevp.type == token.AT or prevp.type == token.DOT: - return NO - - elif p.type == syms.classdef: - if t == token.LPAR: - return NO - - if prev and prev.type == token.LPAR: - return NO - - elif p.type in {syms.subscript, syms.sliceop}: - # indexing - if not prev: - assert p.parent is not None, "subscripts are always parented" - if p.parent.type == syms.subscriptlist: - return SPACE - - return NO - - elif not complex_subscript: - return NO - - elif p.type == syms.atom: - if prev and t == token.DOT: - # dots, but not the first one. - return NO - - elif p.type == syms.dictsetmaker: - # dict unpacking - if prev and prev.type == token.DOUBLESTAR: - return NO - - elif p.type in {syms.factor, syms.star_expr}: - # unary ops - if not prev: - prevp = preceding_leaf(p) - if not prevp or prevp.type in OPENING_BRACKETS: - return NO - - prevp_parent = prevp.parent - assert prevp_parent is not None - if prevp.type == token.COLON and prevp_parent.type in { - syms.subscript, - syms.sliceop, - }: - return NO - - elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument: - return NO - - elif t in {token.NAME, token.NUMBER, token.STRING}: - return NO - - elif p.type == syms.import_from: - if t == token.DOT: - if prev and prev.type == token.DOT: - return NO - - elif t == token.NAME: - if v == "import": - return SPACE - - if prev and prev.type == token.DOT: - return NO - - elif p.type == syms.sliceop: - return NO - - return SPACE - - -def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: - """Return the first leaf that precedes `node`, if any.""" - while node: - res = node.prev_sibling - if res: - if isinstance(res, Leaf): - return res - - try: - return list(res.leaves())[-1] - - except IndexError: - return None - - node = node.parent - return None - - -def prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool: - """Return if the `node` and its previous siblings match types against the provided - list of tokens; the provided `node`has its type matched against the last element in - the list. `None` can be used as the first element to declare that the start of the - list is anchored at the start of its parent's children.""" - if not tokens: - return True - if tokens[-1] is None: - return node is None - if not node: - return False - if node.type != tokens[-1]: - return False - return prev_siblings_are(node.prev_sibling, tokens[:-1]) - - -def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]: - """Return the child of `ancestor` that contains `descendant`.""" - node: Optional[LN] = descendant - while node and node.parent != ancestor: - node = node.parent - return node - - -def container_of(leaf: Leaf) -> LN: - """Return `leaf` or one of its ancestors that is the topmost container of it. - - By "container" we mean a node where `leaf` is the very first child. - """ - same_prefix = leaf.prefix - container: LN = leaf - while container: - parent = container.parent - if parent is None: - break - - if parent.children[0].prefix != same_prefix: - break - - if parent.type == syms.file_input: - break - - if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS: - break - - container = parent - return container - - -def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: - """Return the priority of the `leaf` delimiter, given a line break after it. - - The delimiter priorities returned here are from those delimiters that would - cause a line break after themselves. - - Higher numbers are higher priority. - """ - if leaf.type == token.COMMA: - return COMMA_PRIORITY - - return 0 - - -def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: - """Return the priority of the `leaf` delimiter, given a line break before it. - - The delimiter priorities returned here are from those delimiters that would - cause a line break before themselves. - - Higher numbers are higher priority. - """ - if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS): - # * and ** might also be MATH_OPERATORS but in this case they are not. - # Don't treat them as a delimiter. - return 0 - - if ( - leaf.type == token.DOT - and leaf.parent - and leaf.parent.type not in {syms.import_from, syms.dotted_name} - and (previous is None or previous.type in CLOSING_BRACKETS) - ): - return DOT_PRIORITY - - if ( - leaf.type in MATH_OPERATORS - and leaf.parent - and leaf.parent.type not in {syms.factor, syms.star_expr} - ): - return MATH_PRIORITIES[leaf.type] - - if leaf.type in COMPARATORS: - return COMPARATOR_PRIORITY - - if ( - leaf.type == token.STRING - and previous is not None - and previous.type == token.STRING - ): - return STRING_PRIORITY - - if leaf.type not in {token.NAME, token.ASYNC}: - return 0 - - if ( - leaf.value == "for" - and leaf.parent - and leaf.parent.type in {syms.comp_for, syms.old_comp_for} - or leaf.type == token.ASYNC - ): - if ( - not isinstance(leaf.prev_sibling, Leaf) - or leaf.prev_sibling.value != "async" - ): - return COMPREHENSION_PRIORITY - - if ( - leaf.value == "if" - and leaf.parent - and leaf.parent.type in {syms.comp_if, syms.old_comp_if} - ): - return COMPREHENSION_PRIORITY - - if leaf.value in {"if", "else"} and leaf.parent and leaf.parent.type == syms.test: - return TERNARY_PRIORITY - - if leaf.value == "is": - return COMPARATOR_PRIORITY - - if ( - leaf.value == "in" - and leaf.parent - and leaf.parent.type in {syms.comp_op, syms.comparison} - and not ( - previous is not None - and previous.type == token.NAME - and previous.value == "not" - ) - ): - return COMPARATOR_PRIORITY - - if ( - leaf.value == "not" - and leaf.parent - and leaf.parent.type == syms.comp_op - and not ( - previous is not None - and previous.type == token.NAME - and previous.value == "is" - ) - ): - return COMPARATOR_PRIORITY - - if leaf.value in LOGIC_OPERATORS and leaf.parent: - return LOGIC_PRIORITY - - return 0 - - -FMT_OFF = {"# fmt: off", "# fmt:off", "# yapf: disable"} -FMT_SKIP = {"# fmt: skip", "# fmt:skip"} -FMT_PASS = {*FMT_OFF, *FMT_SKIP} -FMT_ON = {"# fmt: on", "# fmt:on", "# yapf: enable"} - - -def generate_comments(leaf: LN) -> Iterator[Leaf]: - """Clean the prefix of the `leaf` and generate comments from it, if any. - - Comments in lib2to3 are shoved into the whitespace prefix. This happens - in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation - move because it does away with modifying the grammar to include all the - possible places in which comments can be placed. - - The sad consequence for us though is that comments don't "belong" anywhere. - This is why this function generates simple parentless Leaf objects for - comments. We simply don't know what the correct parent should be. - - No matter though, we can live without this. We really only need to - differentiate between inline and standalone comments. The latter don't - share the line with any code. - - Inline comments are emitted as regular token.COMMENT leaves. Standalone - are emitted with a fake STANDALONE_COMMENT token identifier. - """ - for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): - yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) - - -@dataclass -class ProtoComment: - """Describes a piece of syntax that is a comment. - - It's not a :class:`blib2to3.pytree.Leaf` so that: - - * it can be cached (`Leaf` objects should not be reused more than once as - they store their lineno, column, prefix, and parent information); - * `newlines` and `consumed` fields are kept separate from the `value`. This - simplifies handling of special marker comments like ``# fmt: off/on``. - """ - - type: int # token.COMMENT or STANDALONE_COMMENT - value: str # content of the comment - newlines: int # how many newlines before the comment - consumed: int # how many characters of the original leaf's prefix did we consume - - -@lru_cache(maxsize=4096) -def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: - """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" - result: List[ProtoComment] = [] - if not prefix or "#" not in prefix: - return result - - consumed = 0 - nlines = 0 - ignored_lines = 0 - for index, line in enumerate(re.split("\r?\n", prefix)): - consumed += len(line) + 1 # adding the length of the split '\n' - line = line.lstrip() - if not line: - nlines += 1 - if not line.startswith("#"): - # Escaped newlines outside of a comment are not really newlines at - # all. We treat a single-line comment following an escaped newline - # as a simple trailing comment. - if line.endswith("\\"): - ignored_lines += 1 - continue - - if index == ignored_lines and not is_endmarker: - comment_type = token.COMMENT # simple trailing comment - else: - comment_type = STANDALONE_COMMENT - comment = make_comment(line) - result.append( - ProtoComment( - type=comment_type, value=comment, newlines=nlines, consumed=consumed - ) - ) - nlines = 0 - return result - - -def make_comment(content: str) -> str: - """Return a consistently formatted comment from the given `content` string. - - All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single - space between the hash sign and the content. - - If `content` didn't start with a hash sign, one is provided. - """ - content = content.rstrip() - if not content: - return "#" - - if content[0] == "#": - content = content[1:] - NON_BREAKING_SPACE = " " - if ( - content - and content[0] == NON_BREAKING_SPACE - and not content.lstrip().startswith("type:") - ): - content = " " + content[1:] # Replace NBSP by a simple space - if content and content[0] not in " !:#'%": - content = " " + content - return "#" + content - - -def transform_line( - line: Line, mode: Mode, features: Collection[Feature] = () -) -> Iterator[Line]: - """Transform a `line`, potentially splitting it into many lines. - - They should fit in the allotted `line_length` but might not be able to. - - `features` are syntactical features that may be used in the output. - """ - if line.is_comment: - yield line - return - - line_str = line_to_string(line) - - def init_st(ST: Type[StringTransformer]) -> StringTransformer: - """Initialize StringTransformer""" - return ST(mode.line_length, mode.string_normalization) - - string_merge = init_st(StringMerger) - string_paren_strip = init_st(StringParenStripper) - string_split = init_st(StringSplitter) - string_paren_wrap = init_st(StringParenWrapper) - - transformers: List[Transformer] - if ( - not line.contains_uncollapsable_type_comments() - and not line.should_split_rhs - and not line.magic_trailing_comma - and ( - is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) - or line.contains_unsplittable_type_ignore() - ) - and not (line.inside_brackets and line.contains_standalone_comments()) - ): - # Only apply basic string preprocessing, since lines shouldn't be split here. - if mode.experimental_string_processing: - transformers = [string_merge, string_paren_strip] - else: - transformers = [] - elif line.is_def: - transformers = [left_hand_split] - else: - - def rhs(line: Line, features: Collection[Feature]) -> Iterator[Line]: - """Wraps calls to `right_hand_split`. - - The calls increasingly `omit` right-hand trailers (bracket pairs with - content), meaning the trailers get glued together to split on another - bracket pair instead. - """ - for omit in generate_trailers_to_omit(line, mode.line_length): - lines = list( - right_hand_split(line, mode.line_length, features, omit=omit) - ) - # Note: this check is only able to figure out if the first line of the - # *current* transformation fits in the line length. This is true only - # for simple cases. All others require running more transforms via - # `transform_line()`. This check doesn't know if those would succeed. - if is_line_short_enough(lines[0], line_length=mode.line_length): - yield from lines - return - - # All splits failed, best effort split with no omits. - # This mostly happens to multiline strings that are by definition - # reported as not fitting a single line, as well as lines that contain - # trailing commas (those have to be exploded). - yield from right_hand_split( - line, line_length=mode.line_length, features=features - ) - - if mode.experimental_string_processing: - if line.inside_brackets: - transformers = [ - string_merge, - string_paren_strip, - string_split, - delimiter_split, - standalone_comment_split, - string_paren_wrap, - rhs, - ] - else: - transformers = [ - string_merge, - string_paren_strip, - string_split, - string_paren_wrap, - rhs, - ] - else: - if line.inside_brackets: - transformers = [delimiter_split, standalone_comment_split, rhs] - else: - transformers = [rhs] - - for transform in transformers: - # We are accumulating lines in `result` because we might want to abort - # mission and return the original line in the end, or attempt a different - # split altogether. - try: - result = run_transformer(line, transform, mode, features, line_str=line_str) - except CannotTransform: - continue - else: - yield from result - break - - else: - yield line - - -@dataclass # type: ignore -class StringTransformer(ABC): - """ - An implementation of the Transformer protocol that relies on its - subclasses overriding the template methods `do_match(...)` and - `do_transform(...)`. - - This Transformer works exclusively on strings (for example, by merging - or splitting them). - - The following sections can be found among the docstrings of each concrete - StringTransformer subclass. - - Requirements: - Which requirements must be met of the given Line for this - StringTransformer to be applied? - - Transformations: - If the given Line meets all of the above requirements, which string - transformations can you expect to be applied to it by this - StringTransformer? - - Collaborations: - What contractual agreements does this StringTransformer have with other - StringTransfomers? Such collaborations should be eliminated/minimized - as much as possible. - """ - - line_length: int - normalize_strings: bool - __name__ = "StringTransformer" - - @abstractmethod - def do_match(self, line: Line) -> TMatchResult: - """ - Returns: - * Ok(string_idx) such that `line.leaves[string_idx]` is our target - string, if a match was able to be made. - OR - * Err(CannotTransform), if a match was not able to be made. - """ - - @abstractmethod - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: - """ - Yields: - * Ok(new_line) where new_line is the new transformed line. - OR - * Err(CannotTransform) if the transformation failed for some reason. The - `do_match(...)` template method should usually be used to reject - the form of the given Line, but in some cases it is difficult to - know whether or not a Line meets the StringTransformer's - requirements until the transformation is already midway. - - Side Effects: - This method should NOT mutate @line directly, but it MAY mutate the - Line's underlying Node structure. (WARNING: If the underlying Node - structure IS altered, then this method should NOT be allowed to - yield an CannotTransform after that point.) - """ - - def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]: - """ - StringTransformer instances have a call signature that mirrors that of - the Transformer type. - - Raises: - CannotTransform(...) if the concrete StringTransformer class is unable - to transform @line. - """ - # Optimization to avoid calling `self.do_match(...)` when the line does - # not contain any string. - if not any(leaf.type == token.STRING for leaf in line.leaves): - raise CannotTransform("There are no strings in this line.") - - match_result = self.do_match(line) - - if isinstance(match_result, Err): - cant_transform = match_result.err() - raise CannotTransform( - f"The string transformer {self.__class__.__name__} does not recognize" - " this line as one that it can transform." - ) from cant_transform - - string_idx = match_result.ok() - - for line_result in self.do_transform(line, string_idx): - if isinstance(line_result, Err): - cant_transform = line_result.err() - raise CannotTransform( - "StringTransformer failed while attempting to transform string." - ) from cant_transform - line = line_result.ok() - yield line - - -@dataclass -class CustomSplit: - """A custom (i.e. manual) string split. - - A single CustomSplit instance represents a single substring. - - Examples: - Consider the following string: - ``` - "Hi there friend." - " This is a custom" - f" string {split}." - ``` - - This string will correspond to the following three CustomSplit instances: - ``` - CustomSplit(False, 16) - CustomSplit(False, 17) - CustomSplit(True, 16) - ``` - """ - - has_prefix: bool - break_idx: int - - -class CustomSplitMapMixin: - """ - This mixin class is used to map merged strings to a sequence of - CustomSplits, which will then be used to re-split the strings iff none of - the resultant substrings go over the configured max line length. - """ - - _Key = Tuple[StringID, str] - _CUSTOM_SPLIT_MAP: Dict[_Key, Tuple[CustomSplit, ...]] = defaultdict(tuple) - - @staticmethod - def _get_key(string: str) -> "CustomSplitMapMixin._Key": - """ - Returns: - A unique identifier that is used internally to map @string to a - group of custom splits. - """ - return (id(string), string) - - def add_custom_splits( - self, string: str, custom_splits: Iterable[CustomSplit] - ) -> None: - """Custom Split Map Setter Method - - Side Effects: - Adds a mapping from @string to the custom splits @custom_splits. - """ - key = self._get_key(string) - self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits) - - def pop_custom_splits(self, string: str) -> List[CustomSplit]: - """Custom Split Map Getter Method - - Returns: - * A list of the custom splits that are mapped to @string, if any - exist. - OR - * [], otherwise. - - Side Effects: - Deletes the mapping between @string and its associated custom - splits (which are returned to the caller). - """ - key = self._get_key(string) - - custom_splits = self._CUSTOM_SPLIT_MAP[key] - del self._CUSTOM_SPLIT_MAP[key] - - return list(custom_splits) - - def has_custom_splits(self, string: str) -> bool: - """ - Returns: - True iff @string is associated with a set of custom splits. - """ - key = self._get_key(string) - return key in self._CUSTOM_SPLIT_MAP - - -class StringMerger(CustomSplitMapMixin, StringTransformer): - """StringTransformer that merges strings together. - - Requirements: - (A) The line contains adjacent strings such that ALL of the validation checks - listed in StringMerger.__validate_msg(...)'s docstring pass. - OR - (B) The line contains a string which uses line continuation backslashes. - - Transformations: - Depending on which of the two requirements above where met, either: - - (A) The string group associated with the target string is merged. - OR - (B) All line-continuation backslashes are removed from the target string. - - Collaborations: - StringMerger provides custom split information to StringSplitter. - """ - - def do_match(self, line: Line) -> TMatchResult: - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - - for (i, leaf) in enumerate(LL): - if ( - leaf.type == token.STRING - and is_valid_index(i + 1) - and LL[i + 1].type == token.STRING - ): - return Ok(i) - - if leaf.type == token.STRING and "\\\n" in leaf.value: - return Ok(i) - - return TErr("This line has no strings that need merging.") - - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: - new_line = line - rblc_result = self.__remove_backslash_line_continuation_chars( - new_line, string_idx - ) - if isinstance(rblc_result, Ok): - new_line = rblc_result.ok() - - msg_result = self.__merge_string_group(new_line, string_idx) - if isinstance(msg_result, Ok): - new_line = msg_result.ok() - - if isinstance(rblc_result, Err) and isinstance(msg_result, Err): - msg_cant_transform = msg_result.err() - rblc_cant_transform = rblc_result.err() - cant_transform = CannotTransform( - "StringMerger failed to merge any strings in this line." - ) - - # Chain the errors together using `__cause__`. - msg_cant_transform.__cause__ = rblc_cant_transform - cant_transform.__cause__ = msg_cant_transform - - yield Err(cant_transform) - else: - yield Ok(new_line) - - @staticmethod - def __remove_backslash_line_continuation_chars( - line: Line, string_idx: int - ) -> TResult[Line]: - """ - Merge strings that were split across multiple lines using - line-continuation backslashes. - - Returns: - Ok(new_line), if @line contains backslash line-continuation - characters. - OR - Err(CannotTransform), otherwise. - """ - LL = line.leaves - - string_leaf = LL[string_idx] - if not ( - string_leaf.type == token.STRING - and "\\\n" in string_leaf.value - and not has_triple_quotes(string_leaf.value) - ): - return TErr( - f"String leaf {string_leaf} does not contain any backslash line" - " continuation characters." - ) - - new_line = line.clone() - new_line.comments = line.comments.copy() - append_leaves(new_line, line, LL) - - new_string_leaf = new_line.leaves[string_idx] - new_string_leaf.value = new_string_leaf.value.replace("\\\n", "") - - return Ok(new_line) - - def __merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]: - """ - Merges string group (i.e. set of adjacent strings) where the first - string in the group is `line.leaves[string_idx]`. - - Returns: - Ok(new_line), if ALL of the validation checks found in - __validate_msg(...) pass. - OR - Err(CannotTransform), otherwise. - """ - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - - vresult = self.__validate_msg(line, string_idx) - if isinstance(vresult, Err): - return vresult - - # If the string group is wrapped inside an Atom node, we must make sure - # to later replace that Atom with our new (merged) string leaf. - atom_node = LL[string_idx].parent - - # We will place BREAK_MARK in between every two substrings that we - # merge. We will then later go through our final result and use the - # various instances of BREAK_MARK we find to add the right values to - # the custom split map. - BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@" - - QUOTE = LL[string_idx].value[-1] - - def make_naked(string: str, string_prefix: str) -> str: - """Strip @string (i.e. make it a "naked" string) - - Pre-conditions: - * assert_is_leaf_string(@string) - - Returns: - A string that is identical to @string except that - @string_prefix has been stripped, the surrounding QUOTE - characters have been removed, and any remaining QUOTE - characters have been escaped. - """ - assert_is_leaf_string(string) - - RE_EVEN_BACKSLASHES = r"(?:(?= 0 - ), "Logic error while filling the custom string breakpoint cache." - - temp_string = temp_string[mark_idx + len(BREAK_MARK) :] - breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1 - custom_splits.append(CustomSplit(has_prefix, breakpoint_idx)) - - string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, "")) - - if atom_node is not None: - replace_child(atom_node, string_leaf) - - # Build the final line ('new_line') that this method will later return. - new_line = line.clone() - for (i, leaf) in enumerate(LL): - if i == string_idx: - new_line.append(string_leaf) - - if string_idx <= i < string_idx + num_of_strings: - for comment_leaf in line.comments_after(LL[i]): - new_line.append(comment_leaf, preformatted=True) - continue - - append_leaves(new_line, line, [leaf]) - - self.add_custom_splits(string_leaf.value, custom_splits) - return Ok(new_line) - - @staticmethod - def __validate_msg(line: Line, string_idx: int) -> TResult[None]: - """Validate (M)erge (S)tring (G)roup - - Transform-time string validation logic for __merge_string_group(...). - - Returns: - * Ok(None), if ALL validation checks (listed below) pass. - OR - * Err(CannotTransform), if any of the following are true: - - The target string group does not contain ANY stand-alone comments. - - The target string is not in a string group (i.e. it has no - adjacent strings). - - The string group has more than one inline comment. - - The string group has an inline comment that appears to be a pragma. - - The set of all string prefixes in the string group is of - length greater than one and is not equal to {"", "f"}. - - The string group consists of raw strings. - """ - # We first check for "inner" stand-alone comments (i.e. stand-alone - # comments that have a string leaf before them AND after them). - for inc in [1, -1]: - i = string_idx - found_sa_comment = False - is_valid_index = is_valid_index_factory(line.leaves) - while is_valid_index(i) and line.leaves[i].type in [ - token.STRING, - STANDALONE_COMMENT, - ]: - if line.leaves[i].type == STANDALONE_COMMENT: - found_sa_comment = True - elif found_sa_comment: - return TErr( - "StringMerger does NOT merge string groups which contain " - "stand-alone comments." - ) - - i += inc - - num_of_inline_string_comments = 0 - set_of_prefixes = set() - num_of_strings = 0 - for leaf in line.leaves[string_idx:]: - if leaf.type != token.STRING: - # If the string group is trailed by a comma, we count the - # comments trailing the comma to be one of the string group's - # comments. - if leaf.type == token.COMMA and id(leaf) in line.comments: - num_of_inline_string_comments += 1 - break - - if has_triple_quotes(leaf.value): - return TErr("StringMerger does NOT merge multiline strings.") - - num_of_strings += 1 - prefix = get_string_prefix(leaf.value) - if "r" in prefix: - return TErr("StringMerger does NOT merge raw strings.") - - set_of_prefixes.add(prefix) - - if id(leaf) in line.comments: - num_of_inline_string_comments += 1 - if contains_pragma_comment(line.comments[id(leaf)]): - return TErr("Cannot merge strings which have pragma comments.") - - if num_of_strings < 2: - return TErr( - f"Not enough strings to merge (num_of_strings={num_of_strings})." - ) - - if num_of_inline_string_comments > 1: - return TErr( - f"Too many inline string comments ({num_of_inline_string_comments})." - ) - - if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}: - return TErr(f"Too many different prefixes ({set_of_prefixes}).") - - return Ok(None) - - -class StringParenStripper(StringTransformer): - """StringTransformer that strips surrounding parentheses from strings. - - Requirements: - The line contains a string which is surrounded by parentheses and: - - The target string is NOT the only argument to a function call. - - The target string is NOT a "pointless" string. - - If the target string contains a PERCENT, the brackets are not - preceeded or followed by an operator with higher precedence than - PERCENT. - - Transformations: - The parentheses mentioned in the 'Requirements' section are stripped. - - Collaborations: - StringParenStripper has its own inherent usefulness, but it is also - relied on to clean up the parentheses created by StringParenWrapper (in - the event that they are no longer needed). - """ - - def do_match(self, line: Line) -> TMatchResult: - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - - for (idx, leaf) in enumerate(LL): - # Should be a string... - if leaf.type != token.STRING: - continue - - # If this is a "pointless" string... - if ( - leaf.parent - and leaf.parent.parent - and leaf.parent.parent.type == syms.simple_stmt - ): - continue - - # Should be preceded by a non-empty LPAR... - if ( - not is_valid_index(idx - 1) - or LL[idx - 1].type != token.LPAR - or is_empty_lpar(LL[idx - 1]) - ): - continue - - # That LPAR should NOT be preceded by a function name or a closing - # bracket (which could be a function which returns a function or a - # list/dictionary that contains a function)... - if is_valid_index(idx - 2) and ( - LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS - ): - continue - - string_idx = idx - - # Skip the string trailer, if one exists. - string_parser = StringParser() - next_idx = string_parser.parse(LL, string_idx) - - # if the leaves in the parsed string include a PERCENT, we need to - # make sure the initial LPAR is NOT preceded by an operator with - # higher or equal precedence to PERCENT - if is_valid_index(idx - 2): - # mypy can't quite follow unless we name this - before_lpar = LL[idx - 2] - if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and ( - ( - before_lpar.type - in { - token.STAR, - token.AT, - token.SLASH, - token.DOUBLESLASH, - token.PERCENT, - token.TILDE, - token.DOUBLESTAR, - token.AWAIT, - token.LSQB, - token.LPAR, - } - ) - or ( - # only unary PLUS/MINUS - before_lpar.parent - and before_lpar.parent.type == syms.factor - and (before_lpar.type in {token.PLUS, token.MINUS}) - ) - ): - continue - - # Should be followed by a non-empty RPAR... - if ( - is_valid_index(next_idx) - and LL[next_idx].type == token.RPAR - and not is_empty_rpar(LL[next_idx]) - ): - # That RPAR should NOT be followed by anything with higher - # precedence than PERCENT - if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in { - token.DOUBLESTAR, - token.LSQB, - token.LPAR, - token.DOT, - }: - continue - - return Ok(string_idx) - - return TErr("This line has no strings wrapped in parens.") - - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: - LL = line.leaves - - string_parser = StringParser() - rpar_idx = string_parser.parse(LL, string_idx) - - for leaf in (LL[string_idx - 1], LL[rpar_idx]): - if line.comments_after(leaf): - yield TErr( - "Will not strip parentheses which have comments attached to them." - ) - return - - new_line = line.clone() - new_line.comments = line.comments.copy() - try: - append_leaves(new_line, line, LL[: string_idx - 1]) - except BracketMatchError: - # HACK: I believe there is currently a bug somewhere in - # right_hand_split() that is causing brackets to not be tracked - # properly by a shared BracketTracker. - append_leaves(new_line, line, LL[: string_idx - 1], preformatted=True) - - string_leaf = Leaf(token.STRING, LL[string_idx].value) - LL[string_idx - 1].remove() - replace_child(LL[string_idx], string_leaf) - new_line.append(string_leaf) - - append_leaves( - new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :] - ) - - LL[rpar_idx].remove() - - yield Ok(new_line) - - -class BaseStringSplitter(StringTransformer): - """ - Abstract class for StringTransformers which transform a Line's strings by splitting - them or placing them on their own lines where necessary to avoid going over - the configured line length. - - Requirements: - * The target string value is responsible for the line going over the - line length limit. It follows that after all of black's other line - split methods have been exhausted, this line (or one of the resulting - lines after all line splits are performed) would still be over the - line_length limit unless we split this string. - AND - * The target string is NOT a "pointless" string (i.e. a string that has - no parent or siblings). - AND - * The target string is not followed by an inline comment that appears - to be a pragma. - AND - * The target string is not a multiline (i.e. triple-quote) string. - """ - - @abstractmethod - def do_splitter_match(self, line: Line) -> TMatchResult: - """ - BaseStringSplitter asks its clients to override this method instead of - `StringTransformer.do_match(...)`. - - Follows the same protocol as `StringTransformer.do_match(...)`. - - Refer to `help(StringTransformer.do_match)` for more information. - """ - - def do_match(self, line: Line) -> TMatchResult: - match_result = self.do_splitter_match(line) - if isinstance(match_result, Err): - return match_result - - string_idx = match_result.ok() - vresult = self.__validate(line, string_idx) - if isinstance(vresult, Err): - return vresult - - return match_result - - def __validate(self, line: Line, string_idx: int) -> TResult[None]: - """ - Checks that @line meets all of the requirements listed in this classes' - docstring. Refer to `help(BaseStringSplitter)` for a detailed - description of those requirements. - - Returns: - * Ok(None), if ALL of the requirements are met. - OR - * Err(CannotTransform), if ANY of the requirements are NOT met. - """ - LL = line.leaves - - string_leaf = LL[string_idx] - - max_string_length = self.__get_max_string_length(line, string_idx) - if len(string_leaf.value) <= max_string_length: - return TErr( - "The string itself is not what is causing this line to be too long." - ) - - if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [ - token.STRING, - token.NEWLINE, - ]: - return TErr( - f"This string ({string_leaf.value}) appears to be pointless (i.e. has" - " no parent)." - ) - - if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( - line.comments[id(line.leaves[string_idx])] - ): - return TErr( - "Line appears to end with an inline pragma comment. Splitting the line" - " could modify the pragma's behavior." - ) - - if has_triple_quotes(string_leaf.value): - return TErr("We cannot split multiline strings.") - - return Ok(None) - - def __get_max_string_length(self, line: Line, string_idx: int) -> int: - """ - Calculates the max string length used when attempting to determine - whether or not the target string is responsible for causing the line to - go over the line length limit. - - WARNING: This method is tightly coupled to both StringSplitter and - (especially) StringParenWrapper. There is probably a better way to - accomplish what is being done here. - - Returns: - max_string_length: such that `line.leaves[string_idx].value > - max_string_length` implies that the target string IS responsible - for causing this line to exceed the line length limit. - """ - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - - # We use the shorthand "WMA4" in comments to abbreviate "We must - # account for". When giving examples, we use STRING to mean some/any - # valid string. - # - # Finally, we use the following convenience variables: - # - # P: The leaf that is before the target string leaf. - # N: The leaf that is after the target string leaf. - # NN: The leaf that is after N. - - # WMA4 the whitespace at the beginning of the line. - offset = line.depth * 4 - - if is_valid_index(string_idx - 1): - p_idx = string_idx - 1 - if ( - LL[string_idx - 1].type == token.LPAR - and LL[string_idx - 1].value == "" - and string_idx >= 2 - ): - # If the previous leaf is an empty LPAR placeholder, we should skip it. - p_idx -= 1 - - P = LL[p_idx] - if P.type == token.PLUS: - # WMA4 a space and a '+' character (e.g. `+ STRING`). - offset += 2 - - if P.type == token.COMMA: - # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`]. - offset += 3 - - if P.type in [token.COLON, token.EQUAL, token.NAME]: - # This conditional branch is meant to handle dictionary keys, - # variable assignments, 'return STRING' statement lines, and - # 'else STRING' ternary expression lines. - - # WMA4 a single space. - offset += 1 - - # WMA4 the lengths of any leaves that came before that space, - # but after any closing bracket before that space. - for leaf in reversed(LL[: p_idx + 1]): - offset += len(str(leaf)) - if leaf.type in CLOSING_BRACKETS: - break - - if is_valid_index(string_idx + 1): - N = LL[string_idx + 1] - if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2: - # If the next leaf is an empty RPAR placeholder, we should skip it. - N = LL[string_idx + 2] - - if N.type == token.COMMA: - # WMA4 a single comma at the end of the string (e.g `STRING,`). - offset += 1 - - if is_valid_index(string_idx + 2): - NN = LL[string_idx + 2] - - if N.type == token.DOT and NN.type == token.NAME: - # This conditional branch is meant to handle method calls invoked - # off of a string literal up to and including the LPAR character. - - # WMA4 the '.' character. - offset += 1 - - if ( - is_valid_index(string_idx + 3) - and LL[string_idx + 3].type == token.LPAR - ): - # WMA4 the left parenthesis character. - offset += 1 - - # WMA4 the length of the method's name. - offset += len(NN.value) - - has_comments = False - for comment_leaf in line.comments_after(LL[string_idx]): - if not has_comments: - has_comments = True - # WMA4 two spaces before the '#' character. - offset += 2 - - # WMA4 the length of the inline comment. - offset += len(comment_leaf.value) - - max_string_length = self.line_length - offset - return max_string_length - - -class StringSplitter(CustomSplitMapMixin, BaseStringSplitter): - """ - StringTransformer that splits "atom" strings (i.e. strings which exist on - lines by themselves). - - Requirements: - * The line consists ONLY of a single string (with the exception of a - '+' symbol which MAY exist at the start of the line), MAYBE a string - trailer, and MAYBE a trailing comma. - AND - * All of the requirements listed in BaseStringSplitter's docstring. - - Transformations: - The string mentioned in the 'Requirements' section is split into as - many substrings as necessary to adhere to the configured line length. - - In the final set of substrings, no substring should be smaller than - MIN_SUBSTR_SIZE characters. - - The string will ONLY be split on spaces (i.e. each new substring should - start with a space). Note that the string will NOT be split on a space - which is escaped with a backslash. - - If the string is an f-string, it will NOT be split in the middle of an - f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x - else bar()} is an f-expression). - - If the string that is being split has an associated set of custom split - records and those custom splits will NOT result in any line going over - the configured line length, those custom splits are used. Otherwise the - string is split as late as possible (from left-to-right) while still - adhering to the transformation rules listed above. - - Collaborations: - StringSplitter relies on StringMerger to construct the appropriate - CustomSplit objects and add them to the custom split map. - """ - - MIN_SUBSTR_SIZE = 6 - # Matches an "f-expression" (e.g. {var}) that might be found in an f-string. - RE_FEXPR = r""" - (? TMatchResult: - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - - idx = 0 - - # The first leaf MAY be a '+' symbol... - if is_valid_index(idx) and LL[idx].type == token.PLUS: - idx += 1 - - # The next/first leaf MAY be an empty LPAR... - if is_valid_index(idx) and is_empty_lpar(LL[idx]): - idx += 1 - - # The next/first leaf MUST be a string... - if not is_valid_index(idx) or LL[idx].type != token.STRING: - return TErr("Line does not start with a string.") - - string_idx = idx - - # Skip the string trailer, if one exists. - string_parser = StringParser() - idx = string_parser.parse(LL, string_idx) - - # That string MAY be followed by an empty RPAR... - if is_valid_index(idx) and is_empty_rpar(LL[idx]): - idx += 1 - - # That string / empty RPAR leaf MAY be followed by a comma... - if is_valid_index(idx) and LL[idx].type == token.COMMA: - idx += 1 - - # But no more leaves are allowed... - if is_valid_index(idx): - return TErr("This line does not end with a string.") - - return Ok(string_idx) - - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: - LL = line.leaves - - QUOTE = LL[string_idx].value[-1] - - is_valid_index = is_valid_index_factory(LL) - insert_str_child = insert_str_child_factory(LL[string_idx]) - - prefix = get_string_prefix(LL[string_idx].value) - - # We MAY choose to drop the 'f' prefix from substrings that don't - # contain any f-expressions, but ONLY if the original f-string - # contains at least one f-expression. Otherwise, we will alter the AST - # of the program. - drop_pointless_f_prefix = ("f" in prefix) and re.search( - self.RE_FEXPR, LL[string_idx].value, re.VERBOSE - ) - - first_string_line = True - starts_with_plus = LL[0].type == token.PLUS - - def line_needs_plus() -> bool: - return first_string_line and starts_with_plus - - def maybe_append_plus(new_line: Line) -> None: - """ - Side Effects: - If @line starts with a plus and this is the first line we are - constructing, this function appends a PLUS leaf to @new_line - and replaces the old PLUS leaf in the node structure. Otherwise - this function does nothing. - """ - if line_needs_plus(): - plus_leaf = Leaf(token.PLUS, "+") - replace_child(LL[0], plus_leaf) - new_line.append(plus_leaf) - - ends_with_comma = ( - is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA - ) - - def max_last_string() -> int: - """ - Returns: - The max allowed length of the string value used for the last - line we will construct. - """ - result = self.line_length - result -= line.depth * 4 - result -= 1 if ends_with_comma else 0 - result -= 2 if line_needs_plus() else 0 - return result - - # --- Calculate Max Break Index (for string value) - # We start with the line length limit - max_break_idx = self.line_length - # The last index of a string of length N is N-1. - max_break_idx -= 1 - # Leading whitespace is not present in the string value (e.g. Leaf.value). - max_break_idx -= line.depth * 4 - if max_break_idx < 0: - yield TErr( - f"Unable to split {LL[string_idx].value} at such high of a line depth:" - f" {line.depth}" - ) - return - - # Check if StringMerger registered any custom splits. - custom_splits = self.pop_custom_splits(LL[string_idx].value) - # We use them ONLY if none of them would produce lines that exceed the - # line limit. - use_custom_breakpoints = bool( - custom_splits - and all(csplit.break_idx <= max_break_idx for csplit in custom_splits) - ) - - # Temporary storage for the remaining chunk of the string line that - # can't fit onto the line currently being constructed. - rest_value = LL[string_idx].value - - def more_splits_should_be_made() -> bool: - """ - Returns: - True iff `rest_value` (the remaining string value from the last - split), should be split again. - """ - if use_custom_breakpoints: - return len(custom_splits) > 1 - else: - return len(rest_value) > max_last_string() - - string_line_results: List[Ok[Line]] = [] - while more_splits_should_be_made(): - if use_custom_breakpoints: - # Custom User Split (manual) - csplit = custom_splits.pop(0) - break_idx = csplit.break_idx - else: - # Algorithmic Split (automatic) - max_bidx = max_break_idx - 2 if line_needs_plus() else max_break_idx - maybe_break_idx = self.__get_break_idx(rest_value, max_bidx) - if maybe_break_idx is None: - # If we are unable to algorithmically determine a good split - # and this string has custom splits registered to it, we - # fall back to using them--which means we have to start - # over from the beginning. - if custom_splits: - rest_value = LL[string_idx].value - string_line_results = [] - first_string_line = True - use_custom_breakpoints = True - continue - - # Otherwise, we stop splitting here. - break - - break_idx = maybe_break_idx - - # --- Construct `next_value` - next_value = rest_value[:break_idx] + QUOTE - if ( - # Are we allowed to try to drop a pointless 'f' prefix? - drop_pointless_f_prefix - # If we are, will we be successful? - and next_value != self.__normalize_f_string(next_value, prefix) - ): - # If the current custom split did NOT originally use a prefix, - # then `csplit.break_idx` will be off by one after removing - # the 'f' prefix. - break_idx = ( - break_idx + 1 - if use_custom_breakpoints and not csplit.has_prefix - else break_idx - ) - next_value = rest_value[:break_idx] + QUOTE - next_value = self.__normalize_f_string(next_value, prefix) - - # --- Construct `next_leaf` - next_leaf = Leaf(token.STRING, next_value) - insert_str_child(next_leaf) - self.__maybe_normalize_string_quotes(next_leaf) - - # --- Construct `next_line` - next_line = line.clone() - maybe_append_plus(next_line) - next_line.append(next_leaf) - string_line_results.append(Ok(next_line)) - - rest_value = prefix + QUOTE + rest_value[break_idx:] - first_string_line = False - - yield from string_line_results - - if drop_pointless_f_prefix: - rest_value = self.__normalize_f_string(rest_value, prefix) - - rest_leaf = Leaf(token.STRING, rest_value) - insert_str_child(rest_leaf) - - # NOTE: I could not find a test case that verifies that the following - # line is actually necessary, but it seems to be. Otherwise we risk - # not normalizing the last substring, right? - self.__maybe_normalize_string_quotes(rest_leaf) - - last_line = line.clone() - maybe_append_plus(last_line) - - # If there are any leaves to the right of the target string... - if is_valid_index(string_idx + 1): - # We use `temp_value` here to determine how long the last line - # would be if we were to append all the leaves to the right of the - # target string to the last string line. - temp_value = rest_value - for leaf in LL[string_idx + 1 :]: - temp_value += str(leaf) - if leaf.type == token.LPAR: - break - - # Try to fit them all on the same line with the last substring... - if ( - len(temp_value) <= max_last_string() - or LL[string_idx + 1].type == token.COMMA - ): - last_line.append(rest_leaf) - append_leaves(last_line, line, LL[string_idx + 1 :]) - yield Ok(last_line) - # Otherwise, place the last substring on one line and everything - # else on a line below that... - else: - last_line.append(rest_leaf) - yield Ok(last_line) - - non_string_line = line.clone() - append_leaves(non_string_line, line, LL[string_idx + 1 :]) - yield Ok(non_string_line) - # Else the target string was the last leaf... - else: - last_line.append(rest_leaf) - last_line.comments = line.comments.copy() - yield Ok(last_line) - - def __get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]: - """ - This method contains the algorithm that StringSplitter uses to - determine which character to split each string at. - - Args: - @string: The substring that we are attempting to split. - @max_break_idx: The ideal break index. We will return this value if it - meets all the necessary conditions. In the likely event that it - doesn't we will try to find the closest index BELOW @max_break_idx - that does. If that fails, we will expand our search by also - considering all valid indices ABOVE @max_break_idx. - - Pre-Conditions: - * assert_is_leaf_string(@string) - * 0 <= @max_break_idx < len(@string) - - Returns: - break_idx, if an index is able to be found that meets all of the - conditions listed in the 'Transformations' section of this classes' - docstring. - OR - None, otherwise. - """ - is_valid_index = is_valid_index_factory(string) - - assert is_valid_index(max_break_idx) - assert_is_leaf_string(string) - - _fexpr_slices: Optional[List[Tuple[Index, Index]]] = None - - def fexpr_slices() -> Iterator[Tuple[Index, Index]]: - """ - Yields: - All ranges of @string which, if @string were to be split there, - would result in the splitting of an f-expression (which is NOT - allowed). - """ - nonlocal _fexpr_slices - - if _fexpr_slices is None: - _fexpr_slices = [] - for match in re.finditer(self.RE_FEXPR, string, re.VERBOSE): - _fexpr_slices.append(match.span()) - - yield from _fexpr_slices - - is_fstring = "f" in get_string_prefix(string) - - def breaks_fstring_expression(i: Index) -> bool: - """ - Returns: - True iff returning @i would result in the splitting of an - f-expression (which is NOT allowed). - """ - if not is_fstring: - return False - - for (start, end) in fexpr_slices(): - if start <= i < end: - return True - - return False - - def passes_all_checks(i: Index) -> bool: - """ - Returns: - True iff ALL of the conditions listed in the 'Transformations' - section of this classes' docstring would be be met by returning @i. - """ - is_space = string[i] == " " - - is_not_escaped = True - j = i - 1 - while is_valid_index(j) and string[j] == "\\": - is_not_escaped = not is_not_escaped - j -= 1 - - is_big_enough = ( - len(string[i:]) >= self.MIN_SUBSTR_SIZE - and len(string[:i]) >= self.MIN_SUBSTR_SIZE - ) - return ( - is_space - and is_not_escaped - and is_big_enough - and not breaks_fstring_expression(i) - ) - - # First, we check all indices BELOW @max_break_idx. - break_idx = max_break_idx - while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx): - break_idx -= 1 - - if not passes_all_checks(break_idx): - # If that fails, we check all indices ABOVE @max_break_idx. - # - # If we are able to find a valid index here, the next line is going - # to be longer than the specified line length, but it's probably - # better than doing nothing at all. - break_idx = max_break_idx + 1 - while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx): - break_idx += 1 - - if not is_valid_index(break_idx) or not passes_all_checks(break_idx): - return None - - return break_idx - - def __maybe_normalize_string_quotes(self, leaf: Leaf) -> None: - if self.normalize_strings: - normalize_string_quotes(leaf) - - def __normalize_f_string(self, string: str, prefix: str) -> str: - """ - Pre-Conditions: - * assert_is_leaf_string(@string) - - Returns: - * If @string is an f-string that contains no f-expressions, we - return a string identical to @string except that the 'f' prefix - has been stripped and all double braces (i.e. '{{' or '}}') have - been normalized (i.e. turned into '{' or '}'). - OR - * Otherwise, we return @string. - """ - assert_is_leaf_string(string) - - if "f" in prefix and not re.search(self.RE_FEXPR, string, re.VERBOSE): - new_prefix = prefix.replace("f", "") - - temp = string[len(prefix) :] - temp = re.sub(r"\{\{", "{", temp) - temp = re.sub(r"\}\}", "}", temp) - new_string = temp - - return f"{new_prefix}{new_string}" - else: - return string - - -class StringParenWrapper(CustomSplitMapMixin, BaseStringSplitter): - """ - StringTransformer that splits non-"atom" strings (i.e. strings that do not - exist on lines by themselves). - - Requirements: - All of the requirements listed in BaseStringSplitter's docstring in - addition to the requirements listed below: - - * The line is a return/yield statement, which returns/yields a string. - OR - * The line is part of a ternary expression (e.g. `x = y if cond else - z`) such that the line starts with `else `, where is - some string. - OR - * The line is an assert statement, which ends with a string. - OR - * The line is an assignment statement (e.g. `x = ` or `x += - `) such that the variable is being assigned the value of some - string. - OR - * The line is a dictionary key assignment where some valid key is being - assigned the value of some string. - - Transformations: - The chosen string is wrapped in parentheses and then split at the LPAR. - - We then have one line which ends with an LPAR and another line that - starts with the chosen string. The latter line is then split again at - the RPAR. This results in the RPAR (and possibly a trailing comma) - being placed on its own line. - - NOTE: If any leaves exist to the right of the chosen string (except - for a trailing comma, which would be placed after the RPAR), those - leaves are placed inside the parentheses. In effect, the chosen - string is not necessarily being "wrapped" by parentheses. We can, - however, count on the LPAR being placed directly before the chosen - string. - - In other words, StringParenWrapper creates "atom" strings. These - can then be split again by StringSplitter, if necessary. - - Collaborations: - In the event that a string line split by StringParenWrapper is - changed such that it no longer needs to be given its own line, - StringParenWrapper relies on StringParenStripper to clean up the - parentheses it created. - """ - - def do_splitter_match(self, line: Line) -> TMatchResult: - LL = line.leaves - - string_idx = ( - self._return_match(LL) - or self._else_match(LL) - or self._assert_match(LL) - or self._assign_match(LL) - or self._dict_match(LL) - ) - - if string_idx is not None: - string_value = line.leaves[string_idx].value - # If the string has no spaces... - if " " not in string_value: - # And will still violate the line length limit when split... - max_string_length = self.line_length - ((line.depth + 1) * 4) - if len(string_value) > max_string_length: - # And has no associated custom splits... - if not self.has_custom_splits(string_value): - # Then we should NOT put this string on its own line. - return TErr( - "We do not wrap long strings in parentheses when the" - " resultant line would still be over the specified line" - " length and can't be split further by StringSplitter." - ) - return Ok(string_idx) - - return TErr("This line does not contain any non-atomic strings.") - - @staticmethod - def _return_match(LL: List[Leaf]) -> Optional[int]: - """ - Returns: - string_idx such that @LL[string_idx] is equal to our target (i.e. - matched) string, if this line matches the return/yield statement - requirements listed in the 'Requirements' section of this classes' - docstring. - OR - None, otherwise. - """ - # If this line is apart of a return/yield statement and the first leaf - # contains either the "return" or "yield" keywords... - if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[ - 0 - ].value in ["return", "yield"]: - is_valid_index = is_valid_index_factory(LL) - - idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 - # The next visible leaf MUST contain a string... - if is_valid_index(idx) and LL[idx].type == token.STRING: - return idx - - return None - - @staticmethod - def _else_match(LL: List[Leaf]) -> Optional[int]: - """ - Returns: - string_idx such that @LL[string_idx] is equal to our target (i.e. - matched) string, if this line matches the ternary expression - requirements listed in the 'Requirements' section of this classes' - docstring. - OR - None, otherwise. - """ - # If this line is apart of a ternary expression and the first leaf - # contains the "else" keyword... - if ( - parent_type(LL[0]) == syms.test - and LL[0].type == token.NAME - and LL[0].value == "else" - ): - is_valid_index = is_valid_index_factory(LL) - - idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 - # The next visible leaf MUST contain a string... - if is_valid_index(idx) and LL[idx].type == token.STRING: - return idx - - return None - - @staticmethod - def _assert_match(LL: List[Leaf]) -> Optional[int]: - """ - Returns: - string_idx such that @LL[string_idx] is equal to our target (i.e. - matched) string, if this line matches the assert statement - requirements listed in the 'Requirements' section of this classes' - docstring. - OR - None, otherwise. - """ - # If this line is apart of an assert statement and the first leaf - # contains the "assert" keyword... - if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert": - is_valid_index = is_valid_index_factory(LL) - - for (i, leaf) in enumerate(LL): - # We MUST find a comma... - if leaf.type == token.COMMA: - idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 - - # That comma MUST be followed by a string... - if is_valid_index(idx) and LL[idx].type == token.STRING: - string_idx = idx - - # Skip the string trailer, if one exists. - string_parser = StringParser() - idx = string_parser.parse(LL, string_idx) - - # But no more leaves are allowed... - if not is_valid_index(idx): - return string_idx - - return None - - @staticmethod - def _assign_match(LL: List[Leaf]) -> Optional[int]: - """ - Returns: - string_idx such that @LL[string_idx] is equal to our target (i.e. - matched) string, if this line matches the assignment statement - requirements listed in the 'Requirements' section of this classes' - docstring. - OR - None, otherwise. - """ - # If this line is apart of an expression statement or is a function - # argument AND the first leaf contains a variable name... - if ( - parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power] - and LL[0].type == token.NAME - ): - is_valid_index = is_valid_index_factory(LL) - - for (i, leaf) in enumerate(LL): - # We MUST find either an '=' or '+=' symbol... - if leaf.type in [token.EQUAL, token.PLUSEQUAL]: - idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 - - # That symbol MUST be followed by a string... - if is_valid_index(idx) and LL[idx].type == token.STRING: - string_idx = idx - - # Skip the string trailer, if one exists. - string_parser = StringParser() - idx = string_parser.parse(LL, string_idx) - - # The next leaf MAY be a comma iff this line is apart - # of a function argument... - if ( - parent_type(LL[0]) == syms.argument - and is_valid_index(idx) - and LL[idx].type == token.COMMA - ): - idx += 1 - - # But no more leaves are allowed... - if not is_valid_index(idx): - return string_idx - - return None - - @staticmethod - def _dict_match(LL: List[Leaf]) -> Optional[int]: - """ - Returns: - string_idx such that @LL[string_idx] is equal to our target (i.e. - matched) string, if this line matches the dictionary key assignment - statement requirements listed in the 'Requirements' section of this - classes' docstring. - OR - None, otherwise. - """ - # If this line is apart of a dictionary key assignment... - if syms.dictsetmaker in [parent_type(LL[0]), parent_type(LL[0].parent)]: - is_valid_index = is_valid_index_factory(LL) - - for (i, leaf) in enumerate(LL): - # We MUST find a colon... - if leaf.type == token.COLON: - idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 - - # That colon MUST be followed by a string... - if is_valid_index(idx) and LL[idx].type == token.STRING: - string_idx = idx - - # Skip the string trailer, if one exists. - string_parser = StringParser() - idx = string_parser.parse(LL, string_idx) - - # That string MAY be followed by a comma... - if is_valid_index(idx) and LL[idx].type == token.COMMA: - idx += 1 - - # But no more leaves are allowed... - if not is_valid_index(idx): - return string_idx - - return None - - def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: - LL = line.leaves - - is_valid_index = is_valid_index_factory(LL) - insert_str_child = insert_str_child_factory(LL[string_idx]) - - comma_idx = -1 - ends_with_comma = False - if LL[comma_idx].type == token.COMMA: - ends_with_comma = True - - leaves_to_steal_comments_from = [LL[string_idx]] - if ends_with_comma: - leaves_to_steal_comments_from.append(LL[comma_idx]) - - # --- First Line - first_line = line.clone() - left_leaves = LL[:string_idx] - - # We have to remember to account for (possibly invisible) LPAR and RPAR - # leaves that already wrapped the target string. If these leaves do - # exist, we will replace them with our own LPAR and RPAR leaves. - old_parens_exist = False - if left_leaves and left_leaves[-1].type == token.LPAR: - old_parens_exist = True - leaves_to_steal_comments_from.append(left_leaves[-1]) - left_leaves.pop() - - append_leaves(first_line, line, left_leaves) - - lpar_leaf = Leaf(token.LPAR, "(") - if old_parens_exist: - replace_child(LL[string_idx - 1], lpar_leaf) - else: - insert_str_child(lpar_leaf) - first_line.append(lpar_leaf) - - # We throw inline comments that were originally to the right of the - # target string to the top line. They will now be shown to the right of - # the LPAR. - for leaf in leaves_to_steal_comments_from: - for comment_leaf in line.comments_after(leaf): - first_line.append(comment_leaf, preformatted=True) - - yield Ok(first_line) - - # --- Middle (String) Line - # We only need to yield one (possibly too long) string line, since the - # `StringSplitter` will break it down further if necessary. - string_value = LL[string_idx].value - string_line = Line( - mode=line.mode, - depth=line.depth + 1, - inside_brackets=True, - should_split_rhs=line.should_split_rhs, - magic_trailing_comma=line.magic_trailing_comma, - ) - string_leaf = Leaf(token.STRING, string_value) - insert_str_child(string_leaf) - string_line.append(string_leaf) - - old_rpar_leaf = None - if is_valid_index(string_idx + 1): - right_leaves = LL[string_idx + 1 :] - if ends_with_comma: - right_leaves.pop() - - if old_parens_exist: - assert ( - right_leaves and right_leaves[-1].type == token.RPAR - ), "Apparently, old parentheses do NOT exist?!" - old_rpar_leaf = right_leaves.pop() - - append_leaves(string_line, line, right_leaves) - - yield Ok(string_line) - - # --- Last Line - last_line = line.clone() - last_line.bracket_tracker = first_line.bracket_tracker - - new_rpar_leaf = Leaf(token.RPAR, ")") - if old_rpar_leaf is not None: - replace_child(old_rpar_leaf, new_rpar_leaf) - else: - insert_str_child(new_rpar_leaf) - last_line.append(new_rpar_leaf) - - # If the target string ended with a comma, we place this comma to the - # right of the RPAR on the last line. - if ends_with_comma: - comma_leaf = Leaf(token.COMMA, ",") - replace_child(LL[comma_idx], comma_leaf) - last_line.append(comma_leaf) - - yield Ok(last_line) - - -class StringParser: - """ - A state machine that aids in parsing a string's "trailer", which can be - either non-existent, an old-style formatting sequence (e.g. `% varX` or `% - (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX, - varY)`). - - NOTE: A new StringParser object MUST be instantiated for each string - trailer we need to parse. - - Examples: - We shall assume that `line` equals the `Line` object that corresponds - to the following line of python code: - ``` - x = "Some {}.".format("String") + some_other_string - ``` - - Furthermore, we will assume that `string_idx` is some index such that: - ``` - assert line.leaves[string_idx].value == "Some {}." - ``` - - The following code snippet then holds: - ``` - string_parser = StringParser() - idx = string_parser.parse(line.leaves, string_idx) - assert line.leaves[idx].type == token.PLUS - ``` - """ - - DEFAULT_TOKEN = -1 - - # String Parser States - START = 1 - DOT = 2 - NAME = 3 - PERCENT = 4 - SINGLE_FMT_ARG = 5 - LPAR = 6 - RPAR = 7 - DONE = 8 - - # Lookup Table for Next State - _goto: Dict[Tuple[ParserState, NodeType], ParserState] = { - # A string trailer may start with '.' OR '%'. - (START, token.DOT): DOT, - (START, token.PERCENT): PERCENT, - (START, DEFAULT_TOKEN): DONE, - # A '.' MUST be followed by an attribute or method name. - (DOT, token.NAME): NAME, - # A method name MUST be followed by an '(', whereas an attribute name - # is the last symbol in the string trailer. - (NAME, token.LPAR): LPAR, - (NAME, DEFAULT_TOKEN): DONE, - # A '%' symbol can be followed by an '(' or a single argument (e.g. a - # string or variable name). - (PERCENT, token.LPAR): LPAR, - (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG, - # If a '%' symbol is followed by a single argument, that argument is - # the last leaf in the string trailer. - (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE, - # If present, a ')' symbol is the last symbol in a string trailer. - # (NOTE: LPARS and nested RPARS are not included in this lookup table, - # since they are treated as a special case by the parsing logic in this - # classes' implementation.) - (RPAR, DEFAULT_TOKEN): DONE, - } - - def __init__(self) -> None: - self._state = self.START - self._unmatched_lpars = 0 - - def parse(self, leaves: List[Leaf], string_idx: int) -> int: - """ - Pre-conditions: - * @leaves[@string_idx].type == token.STRING - - Returns: - The index directly after the last leaf which is apart of the string - trailer, if a "trailer" exists. - OR - @string_idx + 1, if no string "trailer" exists. - """ - assert leaves[string_idx].type == token.STRING - - idx = string_idx + 1 - while idx < len(leaves) and self._next_state(leaves[idx]): - idx += 1 - return idx - - def _next_state(self, leaf: Leaf) -> bool: - """ - Pre-conditions: - * On the first call to this function, @leaf MUST be the leaf that - was directly after the string leaf in question (e.g. if our target - string is `line.leaves[i]` then the first call to this method must - be `line.leaves[i + 1]`). - * On the next call to this function, the leaf parameter passed in - MUST be the leaf directly following @leaf. - - Returns: - True iff @leaf is apart of the string's trailer. - """ - # We ignore empty LPAR or RPAR leaves. - if is_empty_par(leaf): - return True - - next_token = leaf.type - if next_token == token.LPAR: - self._unmatched_lpars += 1 - - current_state = self._state - - # The LPAR parser state is a special case. We will return True until we - # find the matching RPAR token. - if current_state == self.LPAR: - if next_token == token.RPAR: - self._unmatched_lpars -= 1 - if self._unmatched_lpars == 0: - self._state = self.RPAR - # Otherwise, we use a lookup table to determine the next state. - else: - # If the lookup table matches the current state to the next - # token, we use the lookup table. - if (current_state, next_token) in self._goto: - self._state = self._goto[current_state, next_token] - else: - # Otherwise, we check if a the current state was assigned a - # default. - if (current_state, self.DEFAULT_TOKEN) in self._goto: - self._state = self._goto[current_state, self.DEFAULT_TOKEN] - # If no default has been assigned, then this parser has a logic - # error. - else: - raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!") - - if self._state == self.DONE: - return False - - return True - - -def TErr(err_msg: str) -> Err[CannotTransform]: - """(T)ransform Err - - Convenience function used when working with the TResult type. - """ - cant_transform = CannotTransform(err_msg) - return Err(cant_transform) - - -def contains_pragma_comment(comment_list: List[Leaf]) -> bool: - """ - Returns: - True iff one of the comments in @comment_list is a pragma used by one - of the more common static analysis tools for python (e.g. mypy, flake8, - pylint). - """ - for comment in comment_list: - if comment.value.startswith(("# type:", "# noqa", "# pylint:")): - return True - - return False - - -def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]: - """ - Factory for a convenience function that is used to orphan @string_leaf - and then insert multiple new leaves into the same part of the node - structure that @string_leaf had originally occupied. - - Examples: - Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N = - string_leaf.parent`. Assume the node `N` has the following - original structure: - - Node( - expr_stmt, [ - Leaf(NAME, 'x'), - Leaf(EQUAL, '='), - Leaf(STRING, '"foo"'), - ] - ) - - We then run the code snippet shown below. - ``` - insert_str_child = insert_str_child_factory(string_leaf) - - lpar = Leaf(token.LPAR, '(') - insert_str_child(lpar) - - bar = Leaf(token.STRING, '"bar"') - insert_str_child(bar) - - rpar = Leaf(token.RPAR, ')') - insert_str_child(rpar) - ``` - - After which point, it follows that `string_leaf.parent is None` and - the node `N` now has the following structure: - - Node( - expr_stmt, [ - Leaf(NAME, 'x'), - Leaf(EQUAL, '='), - Leaf(LPAR, '('), - Leaf(STRING, '"bar"'), - Leaf(RPAR, ')'), - ] - ) - """ - string_parent = string_leaf.parent - string_child_idx = string_leaf.remove() - - def insert_str_child(child: LN) -> None: - nonlocal string_child_idx - - assert string_parent is not None - assert string_child_idx is not None - - string_parent.insert_child(string_child_idx, child) - string_child_idx += 1 - - return insert_str_child - - -def has_triple_quotes(string: str) -> bool: - """ - Returns: - True iff @string starts with three quotation characters. - """ - raw_string = string.lstrip(STRING_PREFIX_CHARS) - return raw_string[:3] in {'"""', "'''"} - - -def parent_type(node: Optional[LN]) -> Optional[NodeType]: - """ - Returns: - @node.parent.type, if @node is not None and has a parent. - OR - None, otherwise. - """ - if node is None or node.parent is None: - return None - - return node.parent.type - - -def is_empty_par(leaf: Leaf) -> bool: - return is_empty_lpar(leaf) or is_empty_rpar(leaf) - - -def is_empty_lpar(leaf: Leaf) -> bool: - return leaf.type == token.LPAR and leaf.value == "" - - -def is_empty_rpar(leaf: Leaf) -> bool: - return leaf.type == token.RPAR and leaf.value == "" - - -def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]: - """ - Examples: - ``` - my_list = [1, 2, 3] - - is_valid_index = is_valid_index_factory(my_list) - - assert is_valid_index(0) - assert is_valid_index(2) - - assert not is_valid_index(3) - assert not is_valid_index(-1) - ``` - """ - - def is_valid_index(idx: int) -> bool: - """ - Returns: - True iff @idx is positive AND seq[@idx] does NOT raise an - IndexError. - """ - return 0 <= idx < len(seq) - - return is_valid_index - - -def line_to_string(line: Line) -> str: - """Returns the string representation of @line. - - WARNING: This is known to be computationally expensive. - """ - return str(line).strip("\n") - - -def append_leaves( - new_line: Line, old_line: Line, leaves: List[Leaf], preformatted: bool = False -) -> None: - """ - Append leaves (taken from @old_line) to @new_line, making sure to fix the - underlying Node structure where appropriate. - - All of the leaves in @leaves are duplicated. The duplicates are then - appended to @new_line and used to replace their originals in the underlying - Node structure. Any comments attached to the old leaves are reattached to - the new leaves. - - Pre-conditions: - set(@leaves) is a subset of set(@old_line.leaves). - """ - for old_leaf in leaves: - new_leaf = Leaf(old_leaf.type, old_leaf.value) - replace_child(old_leaf, new_leaf) - new_line.append(new_leaf, preformatted=preformatted) - - for comment_leaf in old_line.comments_after(old_leaf): - new_line.append(comment_leaf, preformatted=True) - - -def replace_child(old_child: LN, new_child: LN) -> None: - """ - Side Effects: - * If @old_child.parent is set, replace @old_child with @new_child in - @old_child's underlying Node structure. - OR - * Otherwise, this function does nothing. - """ - parent = old_child.parent - if not parent: - return - - child_idx = old_child.remove() - if child_idx is not None: - parent.insert_child(child_idx, new_child) - - -def get_string_prefix(string: str) -> str: - """ - Pre-conditions: - * assert_is_leaf_string(@string) - - Returns: - @string's prefix (e.g. '', 'r', 'f', or 'rf'). - """ - assert_is_leaf_string(string) - - prefix = "" - prefix_idx = 0 - while string[prefix_idx] in STRING_PREFIX_CHARS: - prefix += string[prefix_idx].lower() - prefix_idx += 1 - - return prefix - - -def assert_is_leaf_string(string: str) -> None: - """ - Checks the pre-condition that @string has the format that you would expect - of `leaf.value` where `leaf` is some Leaf such that `leaf.type == - token.STRING`. A more precise description of the pre-conditions that are - checked are listed below. - - Pre-conditions: - * @string starts with either ', ", ', or " where - `set()` is some subset of `set(STRING_PREFIX_CHARS)`. - * @string ends with a quote character (' or "). - - Raises: - AssertionError(...) if the pre-conditions listed above are not - satisfied. - """ - dquote_idx = string.find('"') - squote_idx = string.find("'") - if -1 in [dquote_idx, squote_idx]: - quote_idx = max(dquote_idx, squote_idx) - else: - quote_idx = min(squote_idx, dquote_idx) - - assert ( - 0 <= quote_idx < len(string) - 1 - ), f"{string!r} is missing a starting quote character (' or \")." - assert string[-1] in ( - "'", - '"', - ), f"{string!r} is missing an ending quote character (' or \")." - assert set(string[:quote_idx]).issubset( - set(STRING_PREFIX_CHARS) - ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}." - - -def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]: - """Split line into many lines, starting with the first matching bracket pair. - - Note: this usually looks weird, only use this for function definitions. - Prefer RHS otherwise. This is why this function is not symmetrical with - :func:`right_hand_split` which also handles optional parentheses. - """ - tail_leaves: List[Leaf] = [] - body_leaves: List[Leaf] = [] - head_leaves: List[Leaf] = [] - current_leaves = head_leaves - matching_bracket: Optional[Leaf] = None - for leaf in line.leaves: - if ( - current_leaves is body_leaves - and leaf.type in CLOSING_BRACKETS - and leaf.opening_bracket is matching_bracket - ): - current_leaves = tail_leaves if body_leaves else head_leaves - current_leaves.append(leaf) - if current_leaves is head_leaves: - if leaf.type in OPENING_BRACKETS: - matching_bracket = leaf - current_leaves = body_leaves - if not matching_bracket: - raise CannotSplit("No brackets found") - - head = bracket_split_build_line(head_leaves, line, matching_bracket) - body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True) - tail = bracket_split_build_line(tail_leaves, line, matching_bracket) - bracket_split_succeeded_or_raise(head, body, tail) - for result in (head, body, tail): - if result: - yield result - - -def right_hand_split( - line: Line, - line_length: int, - features: Collection[Feature] = (), - omit: Collection[LeafID] = (), -) -> Iterator[Line]: - """Split line into many lines, starting with the last matching bracket pair. - - If the split was by optional parentheses, attempt splitting without them, too. - `omit` is a collection of closing bracket IDs that shouldn't be considered for - this split. - - Note: running this function modifies `bracket_depth` on the leaves of `line`. - """ - tail_leaves: List[Leaf] = [] - body_leaves: List[Leaf] = [] - head_leaves: List[Leaf] = [] - current_leaves = tail_leaves - opening_bracket: Optional[Leaf] = None - closing_bracket: Optional[Leaf] = None - for leaf in reversed(line.leaves): - if current_leaves is body_leaves: - if leaf is opening_bracket: - current_leaves = head_leaves if body_leaves else tail_leaves - current_leaves.append(leaf) - if current_leaves is tail_leaves: - if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit: - opening_bracket = leaf.opening_bracket - closing_bracket = leaf - current_leaves = body_leaves - if not (opening_bracket and closing_bracket and head_leaves): - # If there is no opening or closing_bracket that means the split failed and - # all content is in the tail. Otherwise, if `head_leaves` are empty, it means - # the matching `opening_bracket` wasn't available on `line` anymore. - raise CannotSplit("No brackets found") - - tail_leaves.reverse() - body_leaves.reverse() - head_leaves.reverse() - head = bracket_split_build_line(head_leaves, line, opening_bracket) - body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True) - tail = bracket_split_build_line(tail_leaves, line, opening_bracket) - bracket_split_succeeded_or_raise(head, body, tail) - if ( - Feature.FORCE_OPTIONAL_PARENTHESES not in features - # the opening bracket is an optional paren - and opening_bracket.type == token.LPAR - and not opening_bracket.value - # the closing bracket is an optional paren - and closing_bracket.type == token.RPAR - and not closing_bracket.value - # it's not an import (optional parens are the only thing we can split on - # in this case; attempting a split without them is a waste of time) - and not line.is_import - # there are no standalone comments in the body - and not body.contains_standalone_comments(0) - # and we can actually remove the parens - and can_omit_invisible_parens(body, line_length, omit_on_explode=omit) - ): - omit = {id(closing_bracket), *omit} - try: - yield from right_hand_split(line, line_length, features=features, omit=omit) - return - - except CannotSplit: - if not ( - can_be_split(body) - or is_line_short_enough(body, line_length=line_length) - ): - raise CannotSplit( - "Splitting failed, body is still too long and can't be split." - ) - - elif head.contains_multiline_strings() or tail.contains_multiline_strings(): - raise CannotSplit( - "The current optional pair of parentheses is bound to fail to" - " satisfy the splitting algorithm because the head or the tail" - " contains multiline strings which by definition never fit one" - " line." - ) - - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - for result in (head, body, tail): - if result: - yield result - - -def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: - """Raise :exc:`CannotSplit` if the last left- or right-hand split failed. - - Do nothing otherwise. - - A left- or right-hand split is based on a pair of brackets. Content before - (and including) the opening bracket is left on one line, content inside the - brackets is put on a separate line, and finally content starting with and - following the closing bracket is put on a separate line. - - Those are called `head`, `body`, and `tail`, respectively. If the split - produced the same line (all content in `head`) or ended up with an empty `body` - and the `tail` is just the closing bracket, then it's considered failed. - """ - tail_len = len(str(tail).strip()) - if not body: - if tail_len == 0: - raise CannotSplit("Splitting brackets produced the same line") - - elif tail_len < 3: - raise CannotSplit( - f"Splitting brackets on an empty body to save {tail_len} characters is" - " not worth it" - ) - - -def bracket_split_build_line( - leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False -) -> Line: - """Return a new line with given `leaves` and respective comments from `original`. - - If `is_body` is True, the result line is one-indented inside brackets and as such - has its first leaf's prefix normalized and a trailing comma added when expected. - """ - result = Line(mode=original.mode, depth=original.depth) - if is_body: - result.inside_brackets = True - result.depth += 1 - if leaves: - # Since body is a new indent level, remove spurious leading whitespace. - normalize_prefix(leaves[0], inside_brackets=True) - # Ensure a trailing comma for imports and standalone function arguments, but - # be careful not to add one after any comments or within type annotations. - no_commas = ( - original.is_def - and opening_bracket.value == "(" - and not any(leaf.type == token.COMMA for leaf in leaves) - ) - - if original.is_import or no_commas: - for i in range(len(leaves) - 1, -1, -1): - if leaves[i].type == STANDALONE_COMMENT: - continue - - if leaves[i].type != token.COMMA: - new_comma = Leaf(token.COMMA, ",") - leaves.insert(i + 1, new_comma) - break - - # Populate the line - for leaf in leaves: - result.append(leaf, preformatted=True) - for comment_after in original.comments_after(leaf): - result.append(comment_after, preformatted=True) - if is_body and should_split_line(result, opening_bracket): - result.should_split_rhs = True - return result - - -def dont_increase_indentation(split_func: Transformer) -> Transformer: - """Normalize prefix of the first leaf in every line returned by `split_func`. - - This is a decorator over relevant split functions. - """ - - @wraps(split_func) - def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: - for line in split_func(line, features): - normalize_prefix(line.leaves[0], inside_brackets=True) - yield line - - return split_wrapper - - -@dont_increase_indentation -def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: - """Split according to delimiters of the highest priority. - - If the appropriate Features are given, the split will add trailing commas - also in function signatures and calls that contain `*` and `**`. - """ - try: - last_leaf = line.leaves[-1] - except IndexError: - raise CannotSplit("Line empty") - - bt = line.bracket_tracker - try: - delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)}) - except ValueError: - raise CannotSplit("No delimiters found") - - if delimiter_priority == DOT_PRIORITY: - if bt.delimiter_count_with_priority(delimiter_priority) == 1: - raise CannotSplit("Splitting a single attribute from its owner looks wrong") - - current_line = Line( - mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets - ) - lowest_depth = sys.maxsize - trailing_comma_safe = True - - def append_to_line(leaf: Leaf) -> Iterator[Line]: - """Append `leaf` to current line or to new line if appending impossible.""" - nonlocal current_line - try: - current_line.append_safe(leaf, preformatted=True) - except ValueError: - yield current_line - - current_line = Line( - mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets - ) - current_line.append(leaf) - - for leaf in line.leaves: - yield from append_to_line(leaf) - - for comment_after in line.comments_after(leaf): - yield from append_to_line(comment_after) - - lowest_depth = min(lowest_depth, leaf.bracket_depth) - if leaf.bracket_depth == lowest_depth: - if is_vararg(leaf, within={syms.typedargslist}): - trailing_comma_safe = ( - trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features - ) - elif is_vararg(leaf, within={syms.arglist, syms.argument}): - trailing_comma_safe = ( - trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features - ) - - leaf_priority = bt.delimiters.get(id(leaf)) - if leaf_priority == delimiter_priority: - yield current_line - - current_line = Line( - mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets - ) - if current_line: - if ( - trailing_comma_safe - and delimiter_priority == COMMA_PRIORITY - and current_line.leaves[-1].type != token.COMMA - and current_line.leaves[-1].type != STANDALONE_COMMENT - ): - new_comma = Leaf(token.COMMA, ",") - current_line.append(new_comma) - yield current_line - - -@dont_increase_indentation -def standalone_comment_split( - line: Line, features: Collection[Feature] = () -) -> Iterator[Line]: - """Split standalone comments from the rest of the line.""" - if not line.contains_standalone_comments(0): - raise CannotSplit("Line does not have any standalone comments") - - current_line = Line( - mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets - ) - - def append_to_line(leaf: Leaf) -> Iterator[Line]: - """Append `leaf` to current line or to new line if appending impossible.""" - nonlocal current_line - try: - current_line.append_safe(leaf, preformatted=True) - except ValueError: - yield current_line - - current_line = Line( - line.mode, depth=line.depth, inside_brackets=line.inside_brackets - ) - current_line.append(leaf) - - for leaf in line.leaves: - yield from append_to_line(leaf) - - for comment_after in line.comments_after(leaf): - yield from append_to_line(comment_after) - - if current_line: - yield current_line - - -def is_import(leaf: Leaf) -> bool: - """Return True if the given leaf starts an import statement.""" - p = leaf.parent - t = leaf.type - v = leaf.value - return bool( - t == token.NAME - and ( - (v == "import" and p and p.type == syms.import_name) - or (v == "from" and p and p.type == syms.import_from) - ) - ) - - -def is_type_comment(leaf: Leaf, suffix: str = "") -> bool: - """Return True if the given leaf is a special comment. - Only returns true for type comments for now.""" - t = leaf.type - v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix) - - -def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leave existing extra newlines if not `inside_brackets`. Remove everything - else. - - Note: don't use backslashes for formatting or you'll lose your voting rights. - """ - if not inside_brackets: - spl = leaf.prefix.split("#") - if "\\" not in spl[0]: - nl_count = spl[-1].count("\n") - if len(spl) > 1: - nl_count -= 1 - leaf.prefix = "\n" * nl_count - return - - leaf.prefix = "" - - -def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None: - """Make all string prefixes lowercase. - - If remove_u_prefix is given, also removes any u prefix from the string. - - Note: Mutates its argument. - """ - match = re.match(r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", leaf.value, re.DOTALL) - assert match is not None, f"failed to match string {leaf.value!r}" - orig_prefix = match.group(1) - new_prefix = orig_prefix.replace("F", "f").replace("B", "b").replace("U", "u") - if remove_u_prefix: - new_prefix = new_prefix.replace("u", "") - leaf.value = f"{new_prefix}{match.group(2)}" - - -def normalize_string_quotes(leaf: Leaf) -> None: - """Prefer double quotes but only if it doesn't cause more escaping. - - Adds or removes backslashes as appropriate. Doesn't parse and fix - strings nested in f-strings (yet). - - Note: Mutates its argument. - """ - value = leaf.value.lstrip(STRING_PREFIX_CHARS) - if value[:3] == '"""': - return - - elif value[:3] == "'''": - orig_quote = "'''" - new_quote = '"""' - elif value[0] == '"': - orig_quote = '"' - new_quote = "'" - else: - orig_quote = "'" - new_quote = '"' - first_quote_pos = leaf.value.find(orig_quote) - if first_quote_pos == -1: - return # There's an internal error - - prefix = leaf.value[:first_quote_pos] - unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}") - escaped_new_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}") - escaped_orig_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}") - body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)] - if "r" in prefix.casefold(): - if unescaped_new_quote.search(body): - # There's at least one unescaped new_quote in this raw string - # so converting is impossible - return - - # Do not introduce or remove backslashes in raw strings - new_body = body - else: - # remove unnecessary escapes - new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body) - if body != new_body: - # Consider the string without unnecessary escapes as the original - body = new_body - leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}" - new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body) - new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body) - if "f" in prefix.casefold(): - matches = re.findall( - r""" - (?:[^{]|^)\{ # start of the string or a non-{ followed by a single { - ([^{].*?) # contents of the brackets except if begins with {{ - \}(?:[^}]|$) # A } followed by end of the string or a non-} - """, - new_body, - re.VERBOSE, - ) - for m in matches: - if "\\" in str(m): - # Do not introduce backslashes in interpolated expressions - return - - if new_quote == '"""' and new_body[-1:] == '"': - # edge case: - new_body = new_body[:-1] + '\\"' - orig_escape_count = body.count("\\") - new_escape_count = new_body.count("\\") - if new_escape_count > orig_escape_count: - return # Do not introduce more escaping - - if new_escape_count == orig_escape_count and orig_quote == '"': - return # Prefer double quotes - - leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}" - - -def normalize_numeric_literal(leaf: Leaf) -> None: - """Normalizes numeric (float, int, and complex) literals. - - All letters used in the representation are normalized to lowercase (except - in Python 2 long literals). - """ - text = leaf.value.lower() - if text.startswith(("0o", "0b")): - # Leave octal and binary literals alone. - pass - elif text.startswith("0x"): - text = format_hex(text) - elif "e" in text: - text = format_scientific_notation(text) - elif text.endswith(("j", "l")): - text = format_long_or_complex_number(text) - else: - text = format_float_or_int_string(text) - leaf.value = text - - -def format_hex(text: str) -> str: - """ - Formats a hexadecimal string like "0x12B3" - """ - before, after = text[:2], text[2:] - return f"{before}{after.upper()}" - - -def format_scientific_notation(text: str) -> str: - """Formats a numeric string utilizing scentific notation""" - before, after = text.split("e") - sign = "" - if after.startswith("-"): - after = after[1:] - sign = "-" - elif after.startswith("+"): - after = after[1:] - before = format_float_or_int_string(before) - return f"{before}e{sign}{after}" - - -def format_long_or_complex_number(text: str) -> str: - """Formats a long or complex string like `10L` or `10j`""" - number = text[:-1] - suffix = text[-1] - # Capitalize in "2L" because "l" looks too similar to "1". - if suffix == "l": - suffix = "L" - return f"{format_float_or_int_string(number)}{suffix}" - - -def format_float_or_int_string(text: str) -> str: - """Formats a float string like "1.0".""" - if "." not in text: - return text - - before, after = text.split(".") - return f"{before or 0}.{after or 0}" - - -def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None: - """Make existing optional parentheses invisible or create new ones. - - `parens_after` is a set of string leaf values immediately after which parens - should be put. - - Standardizes on visible parentheses for single-element tuples, and keeps - existing visible parentheses for other tuples and generator expressions. - """ - for pc in list_comments(node.prefix, is_endmarker=False): - if pc.value in FMT_OFF: - # This `node` has a prefix with `# fmt: off`, don't mess with parens. - return - check_lpar = False - for index, child in enumerate(list(node.children)): - # Fixes a bug where invisible parens are not properly stripped from - # assignment statements that contain type annotations. - if isinstance(child, Node) and child.type == syms.annassign: - normalize_invisible_parens(child, parens_after=parens_after) - - # Add parentheses around long tuple unpacking in assignments. - if ( - index == 0 - and isinstance(child, Node) - and child.type == syms.testlist_star_expr - ): - check_lpar = True - - if check_lpar: - if child.type == syms.atom: - if maybe_make_parens_invisible_in_atom(child, parent=node): - wrap_in_parentheses(node, child, visible=False) - elif is_one_tuple(child): - wrap_in_parentheses(node, child, visible=True) - elif node.type == syms.import_from: - # "import from" nodes store parentheses directly as part of - # the statement - if child.type == token.LPAR: - # make parentheses invisible - child.value = "" # type: ignore - node.children[-1].value = "" # type: ignore - elif child.type != token.STAR: - # insert invisible parentheses - node.insert_child(index, Leaf(token.LPAR, "")) - node.append_child(Leaf(token.RPAR, "")) - break - - elif not (isinstance(child, Leaf) and is_multiline_string(child)): - wrap_in_parentheses(node, child, visible=False) - - check_lpar = isinstance(child, Leaf) and child.value in parens_after - - -def normalize_fmt_off(node: Node) -> None: - """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" - try_again = True - while try_again: - try_again = convert_one_fmt_off_pair(node) - - -def convert_one_fmt_off_pair(node: Node) -> bool: - """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. - - Returns True if a pair was converted. - """ - for leaf in node.leaves(): - previous_consumed = 0 - for comment in list_comments(leaf.prefix, is_endmarker=False): - if comment.value not in FMT_PASS: - previous_consumed = comment.consumed - continue - # We only want standalone comments. If there's no previous leaf or - # the previous leaf is indentation, it's a standalone comment in - # disguise. - if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: - prev = preceding_leaf(leaf) - if prev: - if comment.value in FMT_OFF and prev.type not in WHITESPACE: - continue - if comment.value in FMT_SKIP and prev.type in WHITESPACE: - continue - - ignored_nodes = list(generate_ignored_nodes(leaf, comment)) - if not ignored_nodes: - continue - - first = ignored_nodes[0] # Can be a container node with the `leaf`. - parent = first.parent - prefix = first.prefix - first.prefix = prefix[comment.consumed :] - hidden_value = "".join(str(n) for n in ignored_nodes) - if comment.value in FMT_OFF: - hidden_value = comment.value + "\n" + hidden_value - if comment.value in FMT_SKIP: - hidden_value += " " + comment.value - if hidden_value.endswith("\n"): - # That happens when one of the `ignored_nodes` ended with a NEWLINE - # leaf (possibly followed by a DEDENT). - hidden_value = hidden_value[:-1] - first_idx: Optional[int] = None - for ignored in ignored_nodes: - index = ignored.remove() - if first_idx is None: - first_idx = index - assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" - assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" - parent.insert_child( - first_idx, - Leaf( - STANDALONE_COMMENT, - hidden_value, - prefix=prefix[:previous_consumed] + "\n" * comment.newlines, - ), - ) - return True - - return False - - -def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: - """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. - - If comment is skip, returns leaf only. - Stops at the end of the block. - """ - container: Optional[LN] = container_of(leaf) - if comment.value in FMT_SKIP: - prev_sibling = leaf.prev_sibling - if comment.value in leaf.prefix and prev_sibling is not None: - leaf.prefix = leaf.prefix.replace(comment.value, "") - siblings = [prev_sibling] - while ( - "\n" not in prev_sibling.prefix - and prev_sibling.prev_sibling is not None - ): - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - for sibling in siblings: - yield sibling - elif leaf.parent is not None: - yield leaf.parent - return - while container is not None and container.type != token.ENDMARKER: - if is_fmt_on(container): - return - - # fix for fmt: on in children - if contains_fmt_on_at_column(container, leaf.column): - for child in container.children: - if contains_fmt_on_at_column(child, leaf.column): - return - yield child - else: - yield container - container = container.next_sibling - - -def is_fmt_on(container: LN) -> bool: - """Determine whether formatting is switched on within a container. - Determined by whether the last `# fmt:` comment is `on` or `off`. - """ - fmt_on = False - for comment in list_comments(container.prefix, is_endmarker=False): - if comment.value in FMT_ON: - fmt_on = True - elif comment.value in FMT_OFF: - fmt_on = False - return fmt_on - - -def contains_fmt_on_at_column(container: LN, column: int) -> bool: - """Determine if children at a given column have formatting switched on.""" - for child in container.children: - if ( - isinstance(child, Node) - and first_leaf_column(child) == column - or isinstance(child, Leaf) - and child.column == column - ): - if is_fmt_on(child): - return True - - return False - - -def first_leaf_column(node: Node) -> Optional[int]: - """Returns the column of the first leaf child of a node.""" - for child in node.children: - if isinstance(child, Leaf): - return child.column - return None - - -def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: - """If it's safe, make the parens in the atom `node` invisible, recursively. - Additionally, remove repeated, adjacent invisible parens from the atom `node` - as they are redundant. - - Returns whether the node should itself be wrapped in invisible parentheses. - - """ - - if ( - node.type != syms.atom - or is_empty_tuple(node) - or is_one_tuple(node) - or (is_yield(node) and parent.type != syms.expr_stmt) - or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY - ): - return False - - if is_walrus_assignment(node): - if parent.type in [ - syms.annassign, - syms.expr_stmt, - syms.assert_stmt, - syms.return_stmt, - # these ones aren't useful to end users, but they do please fuzzers - syms.for_stmt, - syms.del_stmt, - ]: - return False - - first = node.children[0] - last = node.children[-1] - if first.type == token.LPAR and last.type == token.RPAR: - middle = node.children[1] - # make parentheses invisible - first.value = "" # type: ignore - last.value = "" # type: ignore - maybe_make_parens_invisible_in_atom(middle, parent=parent) - - if is_atom_with_invisible_parens(middle): - # Strip the invisible parens from `middle` by replacing - # it with the child in-between the invisible parens - middle.replace(middle.children[1]) - - return False - - return True - - -def is_atom_with_invisible_parens(node: LN) -> bool: - """Given a `LN`, determines whether it's an atom `node` with invisible - parens. Useful in dedupe-ing and normalizing parens. - """ - if isinstance(node, Leaf) or node.type != syms.atom: - return False - - first, last = node.children[0], node.children[-1] - return ( - isinstance(first, Leaf) - and first.type == token.LPAR - and first.value == "" - and isinstance(last, Leaf) - and last.type == token.RPAR - and last.value == "" - ) - - -def is_empty_tuple(node: LN) -> bool: - """Return True if `node` holds an empty tuple.""" - return ( - node.type == syms.atom - and len(node.children) == 2 - and node.children[0].type == token.LPAR - and node.children[1].type == token.RPAR - ) - - -def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]: - """Returns `wrapped` if `node` is of the shape ( wrapped ). - - Parenthesis can be optional. Returns None otherwise""" - if len(node.children) != 3: - return None - - lpar, wrapped, rpar = node.children - if not (lpar.type == token.LPAR and rpar.type == token.RPAR): - return None - - return wrapped - - -def first_child_is_arith(node: Node) -> bool: - """Whether first child is an arithmetic or a binary arithmetic expression""" - expr_types = { - syms.arith_expr, - syms.shift_expr, - syms.xor_expr, - syms.and_expr, - } - return bool(node.children and node.children[0].type in expr_types) - - -def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: - """Wrap `child` in parentheses. - - This replaces `child` with an atom holding the parentheses and the old - child. That requires moving the prefix. - - If `visible` is False, the leaves will be valueless (and thus invisible). - """ - lpar = Leaf(token.LPAR, "(" if visible else "") - rpar = Leaf(token.RPAR, ")" if visible else "") - prefix = child.prefix - child.prefix = "" - index = child.remove() or 0 - new_child = Node(syms.atom, [lpar, child, rpar]) - new_child.prefix = prefix - parent.insert_child(index, new_child) - - -def is_one_tuple(node: LN) -> bool: - """Return True if `node` holds a tuple with one element, with or without parens.""" - if node.type == syms.atom: - gexp = unwrap_singleton_parenthesis(node) - if gexp is None or gexp.type != syms.testlist_gexp: - return False - - return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA - - return ( - node.type in IMPLICIT_TUPLE - and len(node.children) == 2 - and node.children[1].type == token.COMMA - ) - - -def is_walrus_assignment(node: LN) -> bool: - """Return True iff `node` is of the shape ( test := test )""" - inner = unwrap_singleton_parenthesis(node) - return inner is not None and inner.type == syms.namedexpr_test - - -def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool: - """Return True iff `node` is a trailer valid in a simple decorator""" - return node.type == syms.trailer and ( - ( - len(node.children) == 2 - and node.children[0].type == token.DOT - and node.children[1].type == token.NAME - ) - # last trailer can be an argument-less parentheses pair - or ( - last - and len(node.children) == 2 - and node.children[0].type == token.LPAR - and node.children[1].type == token.RPAR - ) - # last trailer can be arguments - or ( - last - and len(node.children) == 3 - and node.children[0].type == token.LPAR - # and node.children[1].type == syms.argument - and node.children[2].type == token.RPAR - ) - ) - - -def is_simple_decorator_expression(node: LN) -> bool: - """Return True iff `node` could be a 'dotted name' decorator - - This function takes the node of the 'namedexpr_test' of the new decorator - grammar and test if it would be valid under the old decorator grammar. - - The old grammar was: decorator: @ dotted_name [arguments] NEWLINE - The new grammar is : decorator: @ namedexpr_test NEWLINE - """ - if node.type == token.NAME: - return True - if node.type == syms.power: - if node.children: - return ( - node.children[0].type == token.NAME - and all(map(is_simple_decorator_trailer, node.children[1:-1])) - and ( - len(node.children) < 2 - or is_simple_decorator_trailer(node.children[-1], last=True) - ) - ) - return False - - -def is_yield(node: LN) -> bool: - """Return True if `node` holds a `yield` or `yield from` expression.""" - if node.type == syms.yield_expr: - return True - - if node.type == token.NAME and node.value == "yield": # type: ignore - return True - - if node.type != syms.atom: - return False - - if len(node.children) != 3: - return False - - lpar, expr, rpar = node.children - if lpar.type == token.LPAR and rpar.type == token.RPAR: - return is_yield(expr) - - return False - - -def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool: - """Return True if `leaf` is a star or double star in a vararg or kwarg. - - If `within` includes VARARGS_PARENTS, this applies to function signatures. - If `within` includes UNPACKING_PARENTS, it applies to right hand-side - extended iterable unpacking (PEP 3132) and additional unpacking - generalizations (PEP 448). - """ - if leaf.type not in VARARGS_SPECIALS or not leaf.parent: - return False - - p = leaf.parent - if p.type == syms.star_expr: - # Star expressions are also used as assignment targets in extended - # iterable unpacking (PEP 3132). See what its parent is instead. - if not p.parent: - return False - - p = p.parent - - return p.type in within - - -def is_multiline_string(leaf: Leaf) -> bool: - """Return True if `leaf` is a multiline string that actually spans many lines.""" - return has_triple_quotes(leaf.value) and "\n" in leaf.value - - -def is_stub_suite(node: Node) -> bool: - """Return True if `node` is a suite with a stub body.""" - if ( - len(node.children) != 4 - or node.children[0].type != token.NEWLINE - or node.children[1].type != token.INDENT - or node.children[3].type != token.DEDENT - ): - return False - - return is_stub_body(node.children[2]) - - -def is_stub_body(node: LN) -> bool: - """Return True if `node` is a simple statement containing an ellipsis.""" - if not isinstance(node, Node) or node.type != syms.simple_stmt: - return False - - if len(node.children) != 2: - return False - - child = node.children[0] - return ( - child.type == syms.atom - and len(child.children) == 3 - and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) - ) - - -def max_delimiter_priority_in_atom(node: LN) -> Priority: - """Return maximum delimiter priority inside `node`. - - This is specific to atoms with contents contained in a pair of parentheses. - If `node` isn't an atom or there are no enclosing parentheses, returns 0. - """ - if node.type != syms.atom: - return 0 - - first = node.children[0] - last = node.children[-1] - if not (first.type == token.LPAR and last.type == token.RPAR): - return 0 - - bt = BracketTracker() - for c in node.children[1:-1]: - if isinstance(c, Leaf): - bt.mark(c) - else: - for leaf in c.leaves(): - bt.mark(leaf) - try: - return bt.max_delimiter_priority() - - except ValueError: - return 0 - - -def ensure_visible(leaf: Leaf) -> None: - """Make sure parentheses are visible. - - They could be invisible as part of some statements (see - :func:`normalize_invisible_parens` and :func:`visit_import_from`). - """ - if leaf.type == token.LPAR: - leaf.value = "(" - elif leaf.type == token.RPAR: - leaf.value = ")" - - -def should_split_line(line: Line, opening_bracket: Leaf) -> bool: - """Should `line` be immediately split with `delimiter_split()` after RHS?""" - - if not (opening_bracket.parent and opening_bracket.value in "[{("): - return False - - # We're essentially checking if the body is delimited by commas and there's more - # than one of them (we're excluding the trailing comma and if the delimiter priority - # is still commas, that means there's more). - exclude = set() - trailing_comma = False - try: - last_leaf = line.leaves[-1] - if last_leaf.type == token.COMMA: - trailing_comma = True - exclude.add(id(last_leaf)) - max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude) - except (IndexError, ValueError): - return False - - return max_priority == COMMA_PRIORITY and ( - (line.mode.magic_trailing_comma and trailing_comma) - # always explode imports - or opening_bracket.parent.type in {syms.atom, syms.import_from} - ) - - -def is_one_tuple_between(opening: Leaf, closing: Leaf, leaves: List[Leaf]) -> bool: - """Return True if content between `opening` and `closing` looks like a one-tuple.""" - if opening.type != token.LPAR and closing.type != token.RPAR: - return False - - depth = closing.bracket_depth + 1 - for _opening_index, leaf in enumerate(leaves): - if leaf is opening: - break - - else: - raise LookupError("Opening paren not found in `leaves`") - - commas = 0 - _opening_index += 1 - for leaf in leaves[_opening_index:]: - if leaf is closing: - break - - bracket_depth = leaf.bracket_depth - if bracket_depth == depth and leaf.type == token.COMMA: - commas += 1 - if leaf.parent and leaf.parent.type in { - syms.arglist, - syms.typedargslist, - }: - commas += 1 - break - - return commas < 2 - - -def get_features_used(node: Node) -> Set[Feature]: - """Return a set of (relatively) new Python features used in this file. - - Currently looking for: - - f-strings; - - underscores in numeric literals; - - trailing commas after * or ** in function signatures and calls; - - positional only arguments in function signatures and lambdas; - - assignment expression; - - relaxed decorator syntax; - """ - features: Set[Feature] = set() - for n in node.pre_order(): - if n.type == token.STRING: - value_head = n.value[:2] # type: ignore - if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}: - features.add(Feature.F_STRINGS) - - elif n.type == token.NUMBER: - if "_" in n.value: # type: ignore - features.add(Feature.NUMERIC_UNDERSCORES) - - elif n.type == token.SLASH: - if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}: - features.add(Feature.POS_ONLY_ARGUMENTS) - - elif n.type == token.COLONEQUAL: - features.add(Feature.ASSIGNMENT_EXPRESSIONS) - - elif n.type == syms.decorator: - if len(n.children) > 1 and not is_simple_decorator_expression( - n.children[1] - ): - features.add(Feature.RELAXED_DECORATORS) - - elif ( - n.type in {syms.typedargslist, syms.arglist} - and n.children - and n.children[-1].type == token.COMMA - ): - if n.type == syms.typedargslist: - feature = Feature.TRAILING_COMMA_IN_DEF - else: - feature = Feature.TRAILING_COMMA_IN_CALL - - for ch in n.children: - if ch.type in STARS: - features.add(feature) - - if ch.type == syms.argument: - for argch in ch.children: - if argch.type in STARS: - features.add(feature) - - return features - - -def detect_target_versions(node: Node) -> Set[TargetVersion]: - """Detect the version to target based on the nodes used.""" - features = get_features_used(node) - return { - version for version in TargetVersion if features <= VERSION_TO_FEATURES[version] - } - - -def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]: - """Generate sets of closing bracket IDs that should be omitted in a RHS. - - Brackets can be omitted if the entire trailer up to and including - a preceding closing bracket fits in one line. - - Yielded sets are cumulative (contain results of previous yields, too). First - set is empty, unless the line should explode, in which case bracket pairs until - the one that needs to explode are omitted. - """ - - omit: Set[LeafID] = set() - if not line.magic_trailing_comma: - yield omit - - length = 4 * line.depth - opening_bracket: Optional[Leaf] = None - closing_bracket: Optional[Leaf] = None - inner_brackets: Set[LeafID] = set() - for index, leaf, leaf_length in enumerate_with_length(line, reversed=True): - length += leaf_length - if length > line_length: - break - - has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix) - if leaf.type == STANDALONE_COMMENT or has_inline_comment: - break - - if opening_bracket: - if leaf is opening_bracket: - opening_bracket = None - elif leaf.type in CLOSING_BRACKETS: - prev = line.leaves[index - 1] if index > 0 else None - if ( - prev - and prev.type == token.COMMA - and not is_one_tuple_between( - leaf.opening_bracket, leaf, line.leaves - ) - ): - # Never omit bracket pairs with trailing commas. - # We need to explode on those. - break - - inner_brackets.add(id(leaf)) - elif leaf.type in CLOSING_BRACKETS: - prev = line.leaves[index - 1] if index > 0 else None - if prev and prev.type in OPENING_BRACKETS: - # Empty brackets would fail a split so treat them as "inner" - # brackets (e.g. only add them to the `omit` set if another - # pair of brackets was good enough. - inner_brackets.add(id(leaf)) - continue - - if closing_bracket: - omit.add(id(closing_bracket)) - omit.update(inner_brackets) - inner_brackets.clear() - yield omit - - if ( - prev - and prev.type == token.COMMA - and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves) - ): - # Never omit bracket pairs with trailing commas. - # We need to explode on those. - break - - if leaf.value: - opening_bracket = leaf.opening_bracket - closing_bracket = leaf - - -def get_future_imports(node: Node) -> Set[str]: - """Return a set of __future__ imports in the file.""" - imports: Set[str] = set() - - def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]: - for child in children: - if isinstance(child, Leaf): - if child.type == token.NAME: - yield child.value - - elif child.type == syms.import_as_name: - orig_name = child.children[0] - assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports" - assert orig_name.type == token.NAME, "Invalid syntax parsing imports" - yield orig_name.value - - elif child.type == syms.import_as_names: - yield from get_imports_from_children(child.children) - - else: - raise AssertionError("Invalid syntax parsing imports") - - for child in node.children: - if child.type != syms.simple_stmt: - break - - first_child = child.children[0] - if isinstance(first_child, Leaf): - # Continue looking if we see a docstring; otherwise stop. - if ( - len(child.children) == 2 - and first_child.type == token.STRING - and child.children[1].type == token.NEWLINE - ): - continue - - break - - elif first_child.type == syms.import_from: - module_name = first_child.children[1] - if not isinstance(module_name, Leaf) or module_name.value != "__future__": - break - - imports |= set(get_imports_from_children(first_child.children[3:])) - else: - break - - return imports - - -@lru_cache() -def get_gitignore(root: Path) -> PathSpec: - """Return a PathSpec matching gitignore content if present.""" - gitignore = root / ".gitignore" - lines: List[str] = [] - if gitignore.is_file(): - with gitignore.open() as gf: - lines = gf.readlines() - return PathSpec.from_lines("gitwildmatch", lines) - - -def normalize_path_maybe_ignore( - path: Path, root: Path, report: "Report" -) -> Optional[str]: - """Normalize `path`. May return `None` if `path` was ignored. - - `report` is where "path ignored" output goes. - """ - try: - abspath = path if path.is_absolute() else Path.cwd() / path - normalized_path = abspath.resolve().relative_to(root).as_posix() - except OSError as e: - report.path_ignored(path, f"cannot be read because {e}") - return None - - except ValueError: - if path.is_symlink(): - report.path_ignored(path, f"is a symbolic link that points outside {root}") - return None - - raise - - return normalized_path - - -def path_is_excluded( - normalized_path: str, - pattern: Optional[Pattern[str]], -) -> bool: - match = pattern.search(normalized_path) if pattern else None - return bool(match and match.group(0)) - - -def gen_python_files( - paths: Iterable[Path], - root: Path, - include: Pattern[str], - exclude: Pattern[str], - extend_exclude: Optional[Pattern[str]], - force_exclude: Optional[Pattern[str]], - report: "Report", - gitignore: Optional[PathSpec], -) -> Iterator[Path]: - """Generate all files under `path` whose paths are not excluded by the - `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, - but are included by the `include` regex. - - Symbolic links pointing outside of the `root` directory are ignored. - - `report` is where output about exclusions goes. - """ - assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" - for child in paths: - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: - continue - - # First ignore files matching .gitignore, if passed - if gitignore is not None and gitignore.match_file(normalized_path): - report.path_ignored(child, "matches the .gitignore file content") - continue - - # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. - normalized_path = "/" + normalized_path - if child.is_dir(): - normalized_path += "/" - - if path_is_excluded(normalized_path, exclude): - report.path_ignored(child, "matches the --exclude regular expression") - continue - - if path_is_excluded(normalized_path, extend_exclude): - report.path_ignored( - child, "matches the --extend-exclude regular expression" - ) - continue - - if path_is_excluded(normalized_path, force_exclude): - report.path_ignored(child, "matches the --force-exclude regular expression") - continue - - if child.is_dir(): - yield from gen_python_files( - child.iterdir(), - root, - include, - exclude, - extend_exclude, - force_exclude, - report, - gitignore, - ) - - elif child.is_file(): - include_match = include.search(normalized_path) if include else True - if include_match: - yield child - - -@lru_cache() -def find_project_root(srcs: Tuple[str, ...]) -> Path: - """Return a directory containing .git, .hg, or pyproject.toml. - - That directory will be a common parent of all files and directories - passed in `srcs`. - - If no directory in the tree contains a marker that would specify it's the - project root, the root of the file system is returned. - """ - if not srcs: - return Path("/").resolve() - - path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs] - - # A list of lists of parents for each 'src'. 'src' is included as a - # "parent" of itself if it is a directory - src_parents = [ - list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs - ] - - common_base = max( - set.intersection(*(set(parents) for parents in src_parents)), - key=lambda path: path.parts, - ) - - for directory in (common_base, *common_base.parents): - if (directory / ".git").exists(): - return directory - - if (directory / ".hg").is_dir(): - return directory - - if (directory / "pyproject.toml").is_file(): - return directory - - return directory - - -@lru_cache() -def find_user_pyproject_toml() -> Path: - r"""Return the path to the top-level user configuration for black. - - This looks for ~\.black on Windows and ~/.config/black on Linux and other - Unix systems. - """ - if sys.platform == "win32": - # Windows - user_config_path = Path.home() / ".black" - else: - config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config") - user_config_path = Path(config_root).expanduser() / "black" - return user_config_path.resolve() - - -@dataclass -class Report: - """Provides a reformatting counter. Can be rendered with `str(report)`.""" - - check: bool = False - diff: bool = False - quiet: bool = False - verbose: bool = False - change_count: int = 0 - same_count: int = 0 - failure_count: int = 0 - - def done(self, src: Path, changed: Changed) -> None: - """Increment the counter for successful reformatting. Write out a message.""" - if changed is Changed.YES: - reformatted = "would reformat" if self.check or self.diff else "reformatted" - if self.verbose or not self.quiet: - out(f"{reformatted} {src}") - self.change_count += 1 - else: - if self.verbose: - if changed is Changed.NO: - msg = f"{src} already well formatted, good job." - else: - msg = f"{src} wasn't modified on disk since last run." - out(msg, bold=False) - self.same_count += 1 - - def failed(self, src: Path, message: str) -> None: - """Increment the counter for failed reformatting. Write out a message.""" - err(f"error: cannot format {src}: {message}") - self.failure_count += 1 - - def path_ignored(self, path: Path, message: str) -> None: - if self.verbose: - out(f"{path} ignored: {message}", bold=False) - - @property - def return_code(self) -> int: - """Return the exit code that the app should use. - - This considers the current state of changed files and failures: - - if there were any failures, return 123; - - if any files were changed and --check is being used, return 1; - - otherwise return 0. - """ - # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with - # 126 we have special return codes reserved by the shell. - if self.failure_count: - return 123 - - elif self.change_count and self.check: - return 1 - - return 0 - - def __str__(self) -> str: - """Render a color report of the current state. - - Use `click.unstyle` to remove colors. - """ - if self.check or self.diff: - reformatted = "would be reformatted" - unchanged = "would be left unchanged" - failed = "would fail to reformat" - else: - reformatted = "reformatted" - unchanged = "left unchanged" - failed = "failed to reformat" - report = [] - if self.change_count: - s = "s" if self.change_count > 1 else "" - report.append( - click.style(f"{self.change_count} file{s} {reformatted}", bold=True) - ) - if self.same_count: - s = "s" if self.same_count > 1 else "" - report.append(f"{self.same_count} file{s} {unchanged}") - if self.failure_count: - s = "s" if self.failure_count > 1 else "" - report.append( - click.style(f"{self.failure_count} file{s} {failed}", fg="red") - ) - return ", ".join(report) + "." - - -def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]: - filename = "" - if sys.version_info >= (3, 8): - # TODO: support Python 4+ ;) - for minor_version in range(sys.version_info[1], 4, -1): - try: - return ast.parse(src, filename, feature_version=(3, minor_version)) - except SyntaxError: - continue - else: - for feature_version in (7, 6): - try: - return ast3.parse(src, filename, feature_version=feature_version) - except SyntaxError: - continue - if ast27.__name__ == "ast": - raise SyntaxError( - "The requested source code has invalid Python 3 syntax.\n" - "If you are trying to format Python 2 files please reinstall Black" - " with the 'python2' extra: `python3 -m pip install black[python2]`." - ) - return ast27.parse(src) - - -def _fixup_ast_constants( - node: Union[ast.AST, ast3.AST, ast27.AST] -) -> Union[ast.AST, ast3.AST, ast27.AST]: - """Map ast nodes deprecated in 3.8 to Constant.""" - if isinstance(node, (ast.Str, ast3.Str, ast27.Str, ast.Bytes, ast3.Bytes)): - return ast.Constant(value=node.s) - - if isinstance(node, (ast.Num, ast3.Num, ast27.Num)): - return ast.Constant(value=node.n) - - if isinstance(node, (ast.NameConstant, ast3.NameConstant)): - return ast.Constant(value=node.value) - - return node - - -def _stringify_ast( - node: Union[ast.AST, ast3.AST, ast27.AST], depth: int = 0 -) -> Iterator[str]: - """Simple visitor generating strings to compare ASTs by content.""" - - node = _fixup_ast_constants(node) - - yield f"{' ' * depth}{node.__class__.__name__}(" - - for field in sorted(node._fields): # noqa: F402 - # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): - break - - try: - value = getattr(node, field) - except AttributeError: - continue - - yield f"{' ' * (depth+1)}{field}=" - - if isinstance(value, list): - for item in value: - # Ignore nested tuples within del statements, because we may insert - # parentheses and they change the AST. - if ( - field == "targets" - and isinstance(node, (ast.Delete, ast3.Delete, ast27.Delete)) - and isinstance(item, (ast.Tuple, ast3.Tuple, ast27.Tuple)) - ): - for item in item.elts: - yield from _stringify_ast(item, depth + 2) - - elif isinstance(item, (ast.AST, ast3.AST, ast27.AST)): - yield from _stringify_ast(item, depth + 2) - - elif isinstance(value, (ast.AST, ast3.AST, ast27.AST)): - yield from _stringify_ast(value, depth + 2) - - else: - # Constant strings may be indented across newlines, if they are - # docstrings; fold spaces after newlines when comparing. Similarly, - # trailing and leading space may be removed. - # Note that when formatting Python 2 code, at least with Windows - # line-endings, docstrings can end up here as bytes instead of - # str so make sure that we handle both cases. - if ( - isinstance(node, ast.Constant) - and field == "value" - and isinstance(value, (str, bytes)) - ): - lineend = "\n" if isinstance(value, str) else b"\n" - # To normalize, we strip any leading and trailing space from - # each line... - stripped = [line.strip() for line in value.splitlines()] - normalized = lineend.join(stripped) # type: ignore[attr-defined] - # ...and remove any blank lines at the beginning and end of - # the whole string - normalized = normalized.strip() - else: - normalized = value - yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" - - yield f"{' ' * depth}) # /{node.__class__.__name__}" - - -def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None: - """Raise AssertionError if `src` and `dst` aren't equivalent.""" - try: - src_ast = parse_ast(src) - except Exception as exc: - raise AssertionError( - "cannot use --safe with this file; failed to parse source file. AST" - f" error message: {exc}" - ) - - try: - dst_ast = parse_ast(dst) - except Exception as exc: - log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst) - raise AssertionError( - f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. " - "Please report a bug on https://github.com/psf/black/issues. " - f"This invalid output might be helpful: {log}" - ) from None - - src_ast_str = "\n".join(_stringify_ast(src_ast)) - dst_ast_str = "\n".join(_stringify_ast(dst_ast)) - if src_ast_str != dst_ast_str: - log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst")) - raise AssertionError( - "INTERNAL ERROR: Black produced code that is not equivalent to the" - f" source on pass {pass_num}. Please report a bug on " - f"https://github.com/psf/black/issues. This diff might be helpful: {log}" - ) from None - - -def assert_stable(src: str, dst: str, mode: Mode) -> None: - """Raise AssertionError if `dst` reformats differently the second time.""" - newdst = format_str(dst, mode=mode) - if dst != newdst: - log = dump_to_file( - str(mode), - diff(src, dst, "source", "first pass"), - diff(dst, newdst, "first pass", "second pass"), - ) - raise AssertionError( - "INTERNAL ERROR: Black produced different code on the second pass of the" - " formatter. Please report a bug on https://github.com/psf/black/issues." - f" This diff might be helpful: {log}" - ) from None - - -@mypyc_attr(patchable=True) -def dump_to_file(*output: str, ensure_final_newline: bool = True) -> str: - """Dump `output` to a temporary file. Return path to the file.""" - with tempfile.NamedTemporaryFile( - mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8" - ) as f: - for lines in output: - f.write(lines) - if ensure_final_newline and lines and lines[-1] != "\n": - f.write("\n") - return f.name - - -@contextmanager -def nullcontext() -> Iterator[None]: - """Return an empty context manager. - - To be used like `nullcontext` in Python 3.7. - """ - yield - - -def diff(a: str, b: str, a_name: str, b_name: str) -> str: - """Return a unified diff string between strings `a` and `b`.""" - import difflib - - a_lines = [line for line in a.splitlines(keepends=True)] - b_lines = [line for line in b.splitlines(keepends=True)] - diff_lines = [] - for line in difflib.unified_diff( - a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 - ): - # Work around https://bugs.python.org/issue2142 - # See https://www.gnu.org/software/diffutils/manual/html_node/Incomplete-Lines.html - if line[-1] == "\n": - diff_lines.append(line) - else: - diff_lines.append(line + "\n") - diff_lines.append("\\ No newline at end of file\n") - return "".join(diff_lines) - - -def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None: - """asyncio signal handler that cancels all `tasks` and reports to stderr.""" - err("Aborted!") - for task in tasks: - task.cancel() - - -def shutdown(loop: asyncio.AbstractEventLoop) -> None: - """Cancel all pending tasks on `loop`, wait for them, and close the loop.""" - try: - if sys.version_info[:2] >= (3, 7): - all_tasks = asyncio.all_tasks - else: - all_tasks = asyncio.Task.all_tasks - # This part is borrowed from asyncio/runners.py in Python 3.7b2. - to_cancel = [task for task in all_tasks(loop) if not task.done()] - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - loop.run_until_complete( - asyncio.gather(*to_cancel, loop=loop, return_exceptions=True) - ) - finally: - # `concurrent.futures.Future` objects cannot be cancelled once they - # are already running. There might be some when the `shutdown()` happened. - # Silence their logger's spew about the event loop being closed. - cf_logger = logging.getLogger("concurrent.futures") - cf_logger.setLevel(logging.CRITICAL) - loop.close() - - -def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str: - """Replace `regex` with `replacement` twice on `original`. - - This is used by string normalization to perform replaces on - overlapping matches. - """ - return regex.sub(replacement, regex.sub(replacement, original)) - - -def re_compile_maybe_verbose(regex: str) -> Pattern[str]: - """Compile a regular expression string in `regex`. - - If it contains newlines, use verbose mode. - """ - if "\n" in regex: - regex = "(?x)" + regex - compiled: Pattern[str] = re.compile(regex) - return compiled - - -def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]: - """Like `reversed(enumerate(sequence))` if that were possible.""" - index = len(sequence) - 1 - for element in reversed(sequence): - yield (index, element) - index -= 1 - - -def enumerate_with_length( - line: Line, reversed: bool = False -) -> Iterator[Tuple[Index, Leaf, int]]: - """Return an enumeration of leaves with their length. - - Stops prematurely on multiline strings and standalone comments. - """ - op = cast( - Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]], - enumerate_reversed if reversed else enumerate, - ) - for index, leaf in op(line.leaves): - length = len(leaf.prefix) + len(leaf.value) - if "\n" in leaf.value: - return # Multiline strings, we can't continue. - - for comment in line.comments_after(leaf): - length += len(comment.value) - - yield index, leaf, length - - -def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool: - """Return True if `line` is no longer than `line_length`. - - Uses the provided `line_str` rendering, if any, otherwise computes a new one. - """ - if not line_str: - line_str = line_to_string(line) - return ( - len(line_str) <= line_length - and "\n" not in line_str # multiline strings - and not line.contains_standalone_comments() - ) - - -def can_be_split(line: Line) -> bool: - """Return False if the line cannot be split *for sure*. - - This is not an exhaustive search but a cheap heuristic that we can use to - avoid some unfortunate formattings (mostly around wrapping unsplittable code - in unnecessary parentheses). - """ - leaves = line.leaves - if len(leaves) < 2: - return False - - if leaves[0].type == token.STRING and leaves[1].type == token.DOT: - call_count = 0 - dot_count = 0 - next = leaves[-1] - for leaf in leaves[-2::-1]: - if leaf.type in OPENING_BRACKETS: - if next.type not in CLOSING_BRACKETS: - return False - - call_count += 1 - elif leaf.type == token.DOT: - dot_count += 1 - elif leaf.type == token.NAME: - if not (next.type == token.DOT or next.type in OPENING_BRACKETS): - return False - - elif leaf.type not in CLOSING_BRACKETS: - return False - - if dot_count > 1 and call_count > 1: - return False - - return True - - -def can_omit_invisible_parens( - line: Line, - line_length: int, - omit_on_explode: Collection[LeafID] = (), -) -> bool: - """Does `line` have a shape safe to reformat without optional parens around it? - - Returns True for only a subset of potentially nice looking formattings but - the point is to not return false positives that end up producing lines that - are too long. - """ - bt = line.bracket_tracker - if not bt.delimiters: - # Without delimiters the optional parentheses are useless. - return True - - max_priority = bt.max_delimiter_priority() - if bt.delimiter_count_with_priority(max_priority) > 1: - # With more than one delimiter of a kind the optional parentheses read better. - return False - - if max_priority == DOT_PRIORITY: - # A single stranded method call doesn't require optional parentheses. - return True - - assert len(line.leaves) >= 2, "Stranded delimiter" - - # With a single delimiter, omit if the expression starts or ends with - # a bracket. - first = line.leaves[0] - second = line.leaves[1] - if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS: - if _can_omit_opening_paren(line, first=first, line_length=line_length): - return True - - # Note: we are not returning False here because a line might have *both* - # a leading opening bracket and a trailing closing bracket. If the - # opening bracket doesn't match our rule, maybe the closing will. - - penultimate = line.leaves[-2] - last = line.leaves[-1] - if line.magic_trailing_comma: - try: - penultimate, last = last_two_except(line.leaves, omit=omit_on_explode) - except LookupError: - # Turns out we'd omit everything. We cannot skip the optional parentheses. - return False - - if ( - last.type == token.RPAR - or last.type == token.RBRACE - or ( - # don't use indexing for omitting optional parentheses; - # it looks weird - last.type == token.RSQB - and last.parent - and last.parent.type != syms.trailer - ) - ): - if penultimate.type in OPENING_BRACKETS: - # Empty brackets don't help. - return False - - if is_multiline_string(first): - # Additional wrapping of a multiline string in this situation is - # unnecessary. - return True - - if line.magic_trailing_comma and penultimate.type == token.COMMA: - # The rightmost non-omitted bracket pair is the one we want to explode on. - return True - - if _can_omit_closing_paren(line, last=last, line_length=line_length): - return True - - return False - - -def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: - """See `can_omit_invisible_parens`.""" - remainder = False - length = 4 * line.depth - _index = -1 - for _index, leaf, leaf_length in enumerate_with_length(line): - if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: - remainder = True - if remainder: - length += leaf_length - if length > line_length: - break - - if leaf.type in OPENING_BRACKETS: - # There are brackets we can further split on. - remainder = False - - else: - # checked the entire string and line length wasn't exceeded - if len(line.leaves) == _index + 1: - return True - - return False - - -def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: - """See `can_omit_invisible_parens`.""" - length = 4 * line.depth - seen_other_brackets = False - for _index, leaf, leaf_length in enumerate_with_length(line): - length += leaf_length - if leaf is last.opening_bracket: - if seen_other_brackets or length <= line_length: - return True - - elif leaf.type in OPENING_BRACKETS: - # There are brackets we can further split on. - seen_other_brackets = True - - return False - - -def last_two_except(leaves: List[Leaf], omit: Collection[LeafID]) -> Tuple[Leaf, Leaf]: - """Return (penultimate, last) leaves skipping brackets in `omit` and contents.""" - stop_after = None - last = None - for leaf in reversed(leaves): - if stop_after: - if leaf is stop_after: - stop_after = None - continue - - if last: - return leaf, last - - if id(leaf) in omit: - stop_after = leaf.opening_bracket - else: - last = leaf - else: - raise LookupError("Last two leaves were also skipped") - - -def run_transformer( - line: Line, - transform: Transformer, - mode: Mode, - features: Collection[Feature], - *, - line_str: str = "", -) -> List[Line]: - if not line_str: - line_str = line_to_string(line) - result: List[Line] = [] - for transformed_line in transform(line, features): - if str(transformed_line).strip("\n") == line_str: - raise CannotTransform("Line transformer returned an unchanged result") - - result.extend(transform_line(transformed_line, mode=mode, features=features)) - - if not ( - transform.__name__ == "rhs" - and line.bracket_tracker.invisible - and not any(bracket.value for bracket in line.bracket_tracker.invisible) - and not line.contains_multiline_strings() - and not result[0].contains_uncollapsable_type_comments() - and not result[0].contains_unsplittable_type_ignore() - and not is_line_short_enough(result[0], line_length=mode.line_length) - ): - return result - - line_copy = line.clone() - append_leaves(line_copy, line, line.leaves) - features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES} - second_opinion = run_transformer( - line_copy, transform, mode, features_fop, line_str=line_str - ) - if all( - is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion - ): - result = second_opinion - return result - - -def get_cache_file(mode: Mode) -> Path: - return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" - - -def read_cache(mode: Mode) -> Cache: - """Read the cache if it exists and is well formed. - - If it is not well formed, the call to write_cache later should resolve the issue. - """ - cache_file = get_cache_file(mode) - if not cache_file.exists(): - return {} - - with cache_file.open("rb") as fobj: - try: - cache: Cache = pickle.load(fobj) - except (pickle.UnpicklingError, ValueError): - return {} - - return cache - - -def get_cache_info(path: Path) -> CacheInfo: - """Return the information used to check if a file is already formatted or not.""" - stat = path.stat() - return stat.st_mtime, stat.st_size - - -def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: - """Split an iterable of paths in `sources` into two sets. - - The first contains paths of files that modified on disk or are not in the - cache. The other contains paths to non-modified files. - """ - todo, done = set(), set() - for src in sources: - res_src = src.resolve() - if cache.get(str(res_src)) != get_cache_info(res_src): - todo.add(src) - else: - done.add(src) - return todo, done - - -def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None: - """Update the cache file.""" - cache_file = get_cache_file(mode) - try: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - new_cache = { - **cache, - **{str(src.resolve()): get_cache_info(src) for src in sources}, - } - with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: - pickle.dump(new_cache, f, protocol=4) - os.replace(f.name, cache_file) - except OSError: - pass - - -def patch_click() -> None: - """Make Click not crash. - - On certain misconfigured environments, Python 3 selects the ASCII encoding as the - default which restricts paths that it can access during the lifetime of the - application. Click refuses to work in this scenario by raising a RuntimeError. - - In case of Black the likelihood that non-ASCII characters are going to be used in - file paths is minimal since it's Python source code. Moreover, this crash was - spurious on Python 3.7 thanks to PEP 538 and PEP 540. - """ - try: - from click import core - from click import _unicodefun # type: ignore - except ModuleNotFoundError: - return - - for module in (core, _unicodefun): - if hasattr(module, "_verify_python3_env"): - module._verify_python3_env = lambda: None + for module in (core, _unicodefun): + if hasattr(module, "_verify_python3_env"): + module._verify_python3_env = lambda: None def patched_main() -> None: @@ -7025,66 +1056,5 @@ def patched_main() -> None: main() -def is_docstring(leaf: Leaf) -> bool: - if prev_siblings_are( - leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] - ): - return True - - # Multiline docstring on the same line as the `def`. - if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]): - # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python - # grammar. We're safe to return True without further checks. - return True - - return False - - -def lines_with_leading_tabs_expanded(s: str) -> List[str]: - """ - Splits string into lines and expands only leading tabs (following the normal - Python rules) - """ - lines = [] - for line in s.splitlines(): - # Find the index of the first non-whitespace character after a string of - # whitespace that includes at least one tab - match = re.match(r"\s*\t+\s*(\S)", line) - if match: - first_non_whitespace_idx = match.start(1) - - lines.append( - line[:first_non_whitespace_idx].expandtabs() - + line[first_non_whitespace_idx:] - ) - else: - lines.append(line) - return lines - - -def fix_docstring(docstring: str, prefix: str) -> str: - # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation - if not docstring: - return "" - lines = lines_with_leading_tabs_expanded(docstring) - # Determine minimum indentation (first line doesn't count): - indent = sys.maxsize - for line in lines[1:]: - stripped = line.lstrip() - if stripped: - indent = min(indent, len(line) - len(stripped)) - # Remove indentation (first line is special): - trimmed = [lines[0].strip()] - if indent < sys.maxsize: - last_line_idx = len(lines) - 2 - for i, line in enumerate(lines[1:]): - stripped_line = line[indent:].rstrip() - if stripped_line or i == last_line_idx: - trimmed.append(prefix + stripped_line) - else: - trimmed.append("") - return "\n".join(trimmed) - - if __name__ == "__main__": patched_main() diff --git a/src/black/brackets.py b/src/black/brackets.py new file mode 100644 index 00000000000..bb865a0d5b7 --- /dev/null +++ b/src/black/brackets.py @@ -0,0 +1,334 @@ +"""Builds on top of nodes.py to track brackets.""" + +from dataclasses import dataclass, field +import sys +from typing import Dict, Iterable, List, Optional, Tuple, Union + +if sys.version_info < (3, 8): + from typing_extensions import Final +else: + from typing import Final + +from blib2to3.pytree import Leaf, Node +from blib2to3.pgen2 import token + +from black.nodes import syms, is_vararg, VARARGS_PARENTS, UNPACKING_PARENTS +from black.nodes import BRACKET, OPENING_BRACKETS, CLOSING_BRACKETS +from black.nodes import MATH_OPERATORS, COMPARATORS, LOGIC_OPERATORS + +# types +LN = Union[Leaf, Node] +Depth = int +LeafID = int +NodeType = int +Priority = int + + +COMPREHENSION_PRIORITY: Final = 20 +COMMA_PRIORITY: Final = 18 +TERNARY_PRIORITY: Final = 16 +LOGIC_PRIORITY: Final = 14 +STRING_PRIORITY: Final = 12 +COMPARATOR_PRIORITY: Final = 10 +MATH_PRIORITIES: Final = { + token.VBAR: 9, + token.CIRCUMFLEX: 8, + token.AMPER: 7, + token.LEFTSHIFT: 6, + token.RIGHTSHIFT: 6, + token.PLUS: 5, + token.MINUS: 5, + token.STAR: 4, + token.SLASH: 4, + token.DOUBLESLASH: 4, + token.PERCENT: 4, + token.AT: 4, + token.TILDE: 3, + token.DOUBLESTAR: 2, +} +DOT_PRIORITY: Final = 1 + + +class BracketMatchError(KeyError): + """Raised when an opening bracket is unable to be matched to a closing bracket.""" + + +@dataclass +class BracketTracker: + """Keeps track of brackets on a line.""" + + depth: int = 0 + bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = field(default_factory=dict) + delimiters: Dict[LeafID, Priority] = field(default_factory=dict) + previous: Optional[Leaf] = None + _for_loop_depths: List[int] = field(default_factory=list) + _lambda_argument_depths: List[int] = field(default_factory=list) + invisible: List[Leaf] = field(default_factory=list) + + def mark(self, leaf: Leaf) -> None: + """Mark `leaf` with bracket-related metadata. Keep track of delimiters. + + All leaves receive an int `bracket_depth` field that stores how deep + within brackets a given leaf is. 0 means there are no enclosing brackets + that started on this line. + + If a leaf is itself a closing bracket, it receives an `opening_bracket` + field that it forms a pair with. This is a one-directional link to + avoid reference cycles. + + If a leaf is a delimiter (a token on which Black can split the line if + needed) and it's on depth 0, its `id()` is stored in the tracker's + `delimiters` field. + """ + if leaf.type == token.COMMENT: + return + + self.maybe_decrement_after_for_loop_variable(leaf) + self.maybe_decrement_after_lambda_arguments(leaf) + if leaf.type in CLOSING_BRACKETS: + self.depth -= 1 + try: + opening_bracket = self.bracket_match.pop((self.depth, leaf.type)) + except KeyError as e: + raise BracketMatchError( + "Unable to match a closing bracket to the following opening" + f" bracket: {leaf}" + ) from e + leaf.opening_bracket = opening_bracket + if not leaf.value: + self.invisible.append(leaf) + leaf.bracket_depth = self.depth + if self.depth == 0: + delim = is_split_before_delimiter(leaf, self.previous) + if delim and self.previous is not None: + self.delimiters[id(self.previous)] = delim + else: + delim = is_split_after_delimiter(leaf, self.previous) + if delim: + self.delimiters[id(leaf)] = delim + if leaf.type in OPENING_BRACKETS: + self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf + self.depth += 1 + if not leaf.value: + self.invisible.append(leaf) + self.previous = leaf + self.maybe_increment_lambda_arguments(leaf) + self.maybe_increment_for_loop_variable(leaf) + + def any_open_brackets(self) -> bool: + """Return True if there is an yet unmatched open bracket on the line.""" + return bool(self.bracket_match) + + def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> Priority: + """Return the highest priority of a delimiter found on the line. + + Values are consistent with what `is_split_*_delimiter()` return. + Raises ValueError on no delimiters. + """ + return max(v for k, v in self.delimiters.items() if k not in exclude) + + def delimiter_count_with_priority(self, priority: Priority = 0) -> int: + """Return the number of delimiters with the given `priority`. + + If no `priority` is passed, defaults to max priority on the line. + """ + if not self.delimiters: + return 0 + + priority = priority or self.max_delimiter_priority() + return sum(1 for p in self.delimiters.values() if p == priority) + + def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: + """In a for loop, or comprehension, the variables are often unpacks. + + To avoid splitting on the comma in this situation, increase the depth of + tokens between `for` and `in`. + """ + if leaf.type == token.NAME and leaf.value == "for": + self.depth += 1 + self._for_loop_depths.append(self.depth) + return True + + return False + + def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool: + """See `maybe_increment_for_loop_variable` above for explanation.""" + if ( + self._for_loop_depths + and self._for_loop_depths[-1] == self.depth + and leaf.type == token.NAME + and leaf.value == "in" + ): + self.depth -= 1 + self._for_loop_depths.pop() + return True + + return False + + def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool: + """In a lambda expression, there might be more than one argument. + + To avoid splitting on the comma in this situation, increase the depth of + tokens between `lambda` and `:`. + """ + if leaf.type == token.NAME and leaf.value == "lambda": + self.depth += 1 + self._lambda_argument_depths.append(self.depth) + return True + + return False + + def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool: + """See `maybe_increment_lambda_arguments` above for explanation.""" + if ( + self._lambda_argument_depths + and self._lambda_argument_depths[-1] == self.depth + and leaf.type == token.COLON + ): + self.depth -= 1 + self._lambda_argument_depths.pop() + return True + + return False + + def get_open_lsqb(self) -> Optional[Leaf]: + """Return the most recent opening square bracket (if any).""" + return self.bracket_match.get((self.depth - 1, token.RSQB)) + + +def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: + """Return the priority of the `leaf` delimiter, given a line break after it. + + The delimiter priorities returned here are from those delimiters that would + cause a line break after themselves. + + Higher numbers are higher priority. + """ + if leaf.type == token.COMMA: + return COMMA_PRIORITY + + return 0 + + +def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: + """Return the priority of the `leaf` delimiter, given a line break before it. + + The delimiter priorities returned here are from those delimiters that would + cause a line break before themselves. + + Higher numbers are higher priority. + """ + if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS): + # * and ** might also be MATH_OPERATORS but in this case they are not. + # Don't treat them as a delimiter. + return 0 + + if ( + leaf.type == token.DOT + and leaf.parent + and leaf.parent.type not in {syms.import_from, syms.dotted_name} + and (previous is None or previous.type in CLOSING_BRACKETS) + ): + return DOT_PRIORITY + + if ( + leaf.type in MATH_OPERATORS + and leaf.parent + and leaf.parent.type not in {syms.factor, syms.star_expr} + ): + return MATH_PRIORITIES[leaf.type] + + if leaf.type in COMPARATORS: + return COMPARATOR_PRIORITY + + if ( + leaf.type == token.STRING + and previous is not None + and previous.type == token.STRING + ): + return STRING_PRIORITY + + if leaf.type not in {token.NAME, token.ASYNC}: + return 0 + + if ( + leaf.value == "for" + and leaf.parent + and leaf.parent.type in {syms.comp_for, syms.old_comp_for} + or leaf.type == token.ASYNC + ): + if ( + not isinstance(leaf.prev_sibling, Leaf) + or leaf.prev_sibling.value != "async" + ): + return COMPREHENSION_PRIORITY + + if ( + leaf.value == "if" + and leaf.parent + and leaf.parent.type in {syms.comp_if, syms.old_comp_if} + ): + return COMPREHENSION_PRIORITY + + if leaf.value in {"if", "else"} and leaf.parent and leaf.parent.type == syms.test: + return TERNARY_PRIORITY + + if leaf.value == "is": + return COMPARATOR_PRIORITY + + if ( + leaf.value == "in" + and leaf.parent + and leaf.parent.type in {syms.comp_op, syms.comparison} + and not ( + previous is not None + and previous.type == token.NAME + and previous.value == "not" + ) + ): + return COMPARATOR_PRIORITY + + if ( + leaf.value == "not" + and leaf.parent + and leaf.parent.type == syms.comp_op + and not ( + previous is not None + and previous.type == token.NAME + and previous.value == "is" + ) + ): + return COMPARATOR_PRIORITY + + if leaf.value in LOGIC_OPERATORS and leaf.parent: + return LOGIC_PRIORITY + + return 0 + + +def max_delimiter_priority_in_atom(node: LN) -> Priority: + """Return maximum delimiter priority inside `node`. + + This is specific to atoms with contents contained in a pair of parentheses. + If `node` isn't an atom or there are no enclosing parentheses, returns 0. + """ + if node.type != syms.atom: + return 0 + + first = node.children[0] + last = node.children[-1] + if not (first.type == token.LPAR and last.type == token.RPAR): + return 0 + + bt = BracketTracker() + for c in node.children[1:-1]: + if isinstance(c, Leaf): + bt.mark(c) + else: + for leaf in c.leaves(): + bt.mark(leaf) + try: + return bt.max_delimiter_priority() + + except ValueError: + return 0 diff --git a/src/black/cache.py b/src/black/cache.py new file mode 100644 index 00000000000..017a2a91362 --- /dev/null +++ b/src/black/cache.py @@ -0,0 +1,83 @@ +"""Caching of formatted files with feature-based invalidation.""" + +import os +import pickle +from pathlib import Path +import tempfile +from typing import Dict, Iterable, Set, Tuple + +from appdirs import user_cache_dir + +from black.mode import Mode + +from _black_version import version as __version__ + + +# types +Timestamp = float +FileSize = int +CacheInfo = Tuple[Timestamp, FileSize] +Cache = Dict[str, CacheInfo] + + +CACHE_DIR = Path(user_cache_dir("black", version=__version__)) + + +def read_cache(mode: Mode) -> Cache: + """Read the cache if it exists and is well formed. + + If it is not well formed, the call to write_cache later should resolve the issue. + """ + cache_file = get_cache_file(mode) + if not cache_file.exists(): + return {} + + with cache_file.open("rb") as fobj: + try: + cache: Cache = pickle.load(fobj) + except (pickle.UnpicklingError, ValueError): + return {} + + return cache + + +def get_cache_file(mode: Mode) -> Path: + return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" + + +def get_cache_info(path: Path) -> CacheInfo: + """Return the information used to check if a file is already formatted or not.""" + stat = path.stat() + return stat.st_mtime, stat.st_size + + +def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: + """Split an iterable of paths in `sources` into two sets. + + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. + """ + todo, done = set(), set() + for src in sources: + res_src = src.resolve() + if cache.get(str(res_src)) != get_cache_info(res_src): + todo.add(src) + else: + done.add(src) + return todo, done + + +def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None: + """Update the cache file.""" + cache_file = get_cache_file(mode) + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + new_cache = { + **cache, + **{str(src.resolve()): get_cache_info(src) for src in sources}, + } + with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: + pickle.dump(new_cache, f, protocol=4) + os.replace(f.name, cache_file) + except OSError: + pass diff --git a/src/black/comments.py b/src/black/comments.py new file mode 100644 index 00000000000..415e391b2a7 --- /dev/null +++ b/src/black/comments.py @@ -0,0 +1,269 @@ +from dataclasses import dataclass +from functools import lru_cache +import regex as re +from typing import Iterator, List, Optional, Union + +from blib2to3.pytree import Node, Leaf +from blib2to3.pgen2 import token + +from black.nodes import first_leaf_column, preceding_leaf, container_of +from black.nodes import STANDALONE_COMMENT, WHITESPACE + +# types +LN = Union[Leaf, Node] + + +FMT_OFF = {"# fmt: off", "# fmt:off", "# yapf: disable"} +FMT_SKIP = {"# fmt: skip", "# fmt:skip"} +FMT_PASS = {*FMT_OFF, *FMT_SKIP} +FMT_ON = {"# fmt: on", "# fmt:on", "# yapf: enable"} + + +@dataclass +class ProtoComment: + """Describes a piece of syntax that is a comment. + + It's not a :class:`blib2to3.pytree.Leaf` so that: + + * it can be cached (`Leaf` objects should not be reused more than once as + they store their lineno, column, prefix, and parent information); + * `newlines` and `consumed` fields are kept separate from the `value`. This + simplifies handling of special marker comments like ``# fmt: off/on``. + """ + + type: int # token.COMMENT or STANDALONE_COMMENT + value: str # content of the comment + newlines: int # how many newlines before the comment + consumed: int # how many characters of the original leaf's prefix did we consume + + +def generate_comments(leaf: LN) -> Iterator[Leaf]: + """Clean the prefix of the `leaf` and generate comments from it, if any. + + Comments in lib2to3 are shoved into the whitespace prefix. This happens + in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation + move because it does away with modifying the grammar to include all the + possible places in which comments can be placed. + + The sad consequence for us though is that comments don't "belong" anywhere. + This is why this function generates simple parentless Leaf objects for + comments. We simply don't know what the correct parent should be. + + No matter though, we can live without this. We really only need to + differentiate between inline and standalone comments. The latter don't + share the line with any code. + + Inline comments are emitted as regular token.COMMENT leaves. Standalone + are emitted with a fake STANDALONE_COMMENT token identifier. + """ + for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): + yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) + + +@lru_cache(maxsize=4096) +def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: + """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" + result: List[ProtoComment] = [] + if not prefix or "#" not in prefix: + return result + + consumed = 0 + nlines = 0 + ignored_lines = 0 + for index, line in enumerate(re.split("\r?\n", prefix)): + consumed += len(line) + 1 # adding the length of the split '\n' + line = line.lstrip() + if not line: + nlines += 1 + if not line.startswith("#"): + # Escaped newlines outside of a comment are not really newlines at + # all. We treat a single-line comment following an escaped newline + # as a simple trailing comment. + if line.endswith("\\"): + ignored_lines += 1 + continue + + if index == ignored_lines and not is_endmarker: + comment_type = token.COMMENT # simple trailing comment + else: + comment_type = STANDALONE_COMMENT + comment = make_comment(line) + result.append( + ProtoComment( + type=comment_type, value=comment, newlines=nlines, consumed=consumed + ) + ) + nlines = 0 + return result + + +def make_comment(content: str) -> str: + """Return a consistently formatted comment from the given `content` string. + + All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single + space between the hash sign and the content. + + If `content` didn't start with a hash sign, one is provided. + """ + content = content.rstrip() + if not content: + return "#" + + if content[0] == "#": + content = content[1:] + NON_BREAKING_SPACE = " " + if ( + content + and content[0] == NON_BREAKING_SPACE + and not content.lstrip().startswith("type:") + ): + content = " " + content[1:] # Replace NBSP by a simple space + if content and content[0] not in " !:#'%": + content = " " + content + return "#" + content + + +def normalize_fmt_off(node: Node) -> None: + """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" + try_again = True + while try_again: + try_again = convert_one_fmt_off_pair(node) + + +def convert_one_fmt_off_pair(node: Node) -> bool: + """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. + + Returns True if a pair was converted. + """ + for leaf in node.leaves(): + previous_consumed = 0 + for comment in list_comments(leaf.prefix, is_endmarker=False): + if comment.value not in FMT_PASS: + previous_consumed = comment.consumed + continue + # We only want standalone comments. If there's no previous leaf or + # the previous leaf is indentation, it's a standalone comment in + # disguise. + if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: + prev = preceding_leaf(leaf) + if prev: + if comment.value in FMT_OFF and prev.type not in WHITESPACE: + continue + if comment.value in FMT_SKIP and prev.type in WHITESPACE: + continue + + ignored_nodes = list(generate_ignored_nodes(leaf, comment)) + if not ignored_nodes: + continue + + first = ignored_nodes[0] # Can be a container node with the `leaf`. + parent = first.parent + prefix = first.prefix + first.prefix = prefix[comment.consumed :] + hidden_value = "".join(str(n) for n in ignored_nodes) + if comment.value in FMT_OFF: + hidden_value = comment.value + "\n" + hidden_value + if comment.value in FMT_SKIP: + hidden_value += " " + comment.value + if hidden_value.endswith("\n"): + # That happens when one of the `ignored_nodes` ended with a NEWLINE + # leaf (possibly followed by a DEDENT). + hidden_value = hidden_value[:-1] + first_idx: Optional[int] = None + for ignored in ignored_nodes: + index = ignored.remove() + if first_idx is None: + first_idx = index + assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" + assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" + parent.insert_child( + first_idx, + Leaf( + STANDALONE_COMMENT, + hidden_value, + prefix=prefix[:previous_consumed] + "\n" * comment.newlines, + ), + ) + return True + + return False + + +def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: + """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. + + If comment is skip, returns leaf only. + Stops at the end of the block. + """ + container: Optional[LN] = container_of(leaf) + if comment.value in FMT_SKIP: + prev_sibling = leaf.prev_sibling + if comment.value in leaf.prefix and prev_sibling is not None: + leaf.prefix = leaf.prefix.replace(comment.value, "") + siblings = [prev_sibling] + while ( + "\n" not in prev_sibling.prefix + and prev_sibling.prev_sibling is not None + ): + prev_sibling = prev_sibling.prev_sibling + siblings.insert(0, prev_sibling) + for sibling in siblings: + yield sibling + elif leaf.parent is not None: + yield leaf.parent + return + while container is not None and container.type != token.ENDMARKER: + if is_fmt_on(container): + return + + # fix for fmt: on in children + if contains_fmt_on_at_column(container, leaf.column): + for child in container.children: + if contains_fmt_on_at_column(child, leaf.column): + return + yield child + else: + yield container + container = container.next_sibling + + +def is_fmt_on(container: LN) -> bool: + """Determine whether formatting is switched on within a container. + Determined by whether the last `# fmt:` comment is `on` or `off`. + """ + fmt_on = False + for comment in list_comments(container.prefix, is_endmarker=False): + if comment.value in FMT_ON: + fmt_on = True + elif comment.value in FMT_OFF: + fmt_on = False + return fmt_on + + +def contains_fmt_on_at_column(container: LN, column: int) -> bool: + """Determine if children at a given column have formatting switched on.""" + for child in container.children: + if ( + isinstance(child, Node) + and first_leaf_column(child) == column + or isinstance(child, Leaf) + and child.column == column + ): + if is_fmt_on(child): + return True + + return False + + +def contains_pragma_comment(comment_list: List[Leaf]) -> bool: + """ + Returns: + True iff one of the comments in @comment_list is a pragma used by one + of the more common static analysis tools for python (e.g. mypy, flake8, + pylint). + """ + for comment in comment_list: + if comment.value.startswith(("# type:", "# noqa", "# pylint:")): + return True + + return False diff --git a/src/black/concurrency.py b/src/black/concurrency.py new file mode 100644 index 00000000000..119a9a71faf --- /dev/null +++ b/src/black/concurrency.py @@ -0,0 +1,39 @@ +import asyncio +import logging +import sys +from typing import Any, Iterable + +from black.output import err + + +def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None: + """asyncio signal handler that cancels all `tasks` and reports to stderr.""" + err("Aborted!") + for task in tasks: + task.cancel() + + +def shutdown(loop: asyncio.AbstractEventLoop) -> None: + """Cancel all pending tasks on `loop`, wait for them, and close the loop.""" + try: + if sys.version_info[:2] >= (3, 7): + all_tasks = asyncio.all_tasks + else: + all_tasks = asyncio.Task.all_tasks + # This part is borrowed from asyncio/runners.py in Python 3.7b2. + to_cancel = [task for task in all_tasks(loop) if not task.done()] + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + loop.run_until_complete( + asyncio.gather(*to_cancel, loop=loop, return_exceptions=True) + ) + finally: + # `concurrent.futures.Future` objects cannot be cancelled once they + # are already running. There might be some when the `shutdown()` happened. + # Silence their logger's spew about the event loop being closed. + cf_logger = logging.getLogger("concurrent.futures") + cf_logger.setLevel(logging.CRITICAL) + loop.close() diff --git a/src/black/const.py b/src/black/const.py new file mode 100644 index 00000000000..821258588ab --- /dev/null +++ b/src/black/const.py @@ -0,0 +1,4 @@ +DEFAULT_LINE_LENGTH = 88 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 +DEFAULT_INCLUDES = r"\.pyi?$" +STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/debug.py b/src/black/debug.py new file mode 100644 index 00000000000..5143076ab35 --- /dev/null +++ b/src/black/debug.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass +from typing import Iterator, TypeVar, Union + +from blib2to3.pytree import Node, Leaf, type_repr +from blib2to3.pgen2 import token + +from black.nodes import Visitor +from black.output import out +from black.parsing import lib2to3_parse + +LN = Union[Leaf, Node] +T = TypeVar("T") + + +@dataclass +class DebugVisitor(Visitor[T]): + tree_depth: int = 0 + + def visit_default(self, node: LN) -> Iterator[T]: + indent = " " * (2 * self.tree_depth) + if isinstance(node, Node): + _type = type_repr(node.type) + out(f"{indent}{_type}", fg="yellow") + self.tree_depth += 1 + for child in node.children: + yield from self.visit(child) + + self.tree_depth -= 1 + out(f"{indent}/{_type}", fg="yellow", bold=False) + else: + _type = token.tok_name.get(node.type, str(node.type)) + out(f"{indent}{_type}", fg="blue", nl=False) + if node.prefix: + # We don't have to handle prefixes for `Node` objects since + # that delegates to the first child anyway. + out(f" {node.prefix!r}", fg="green", bold=False, nl=False) + out(f" {node.value!r}", fg="blue", bold=False) + + @classmethod + def show(cls, code: Union[str, Leaf, Node]) -> None: + """Pretty-print the lib2to3 AST of a given string of `code`. + + Convenience method for debugging. + """ + v: DebugVisitor[None] = DebugVisitor() + if isinstance(code, str): + code = lib2to3_parse(code) + list(v.visit(code)) diff --git a/src/black/files.py b/src/black/files.py new file mode 100644 index 00000000000..1be560643a1 --- /dev/null +++ b/src/black/files.py @@ -0,0 +1,241 @@ +from functools import lru_cache +import io +import os +from pathlib import Path +import sys +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Optional, + Pattern, + Sequence, + Tuple, + Union, + TYPE_CHECKING, +) + +from pathspec import PathSpec +import toml + +from black.output import err +from black.report import Report + +if TYPE_CHECKING: + import colorama # noqa: F401 + + +@lru_cache() +def find_project_root(srcs: Sequence[str]) -> Path: + """Return a directory containing .git, .hg, or pyproject.toml. + + That directory will be a common parent of all files and directories + passed in `srcs`. + + If no directory in the tree contains a marker that would specify it's the + project root, the root of the file system is returned. + """ + if not srcs: + return Path("/").resolve() + + path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs] + + # A list of lists of parents for each 'src'. 'src' is included as a + # "parent" of itself if it is a directory + src_parents = [ + list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs + ] + + common_base = max( + set.intersection(*(set(parents) for parents in src_parents)), + key=lambda path: path.parts, + ) + + for directory in (common_base, *common_base.parents): + if (directory / ".git").exists(): + return directory + + if (directory / ".hg").is_dir(): + return directory + + if (directory / "pyproject.toml").is_file(): + return directory + + return directory + + +def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: + """Find the absolute filepath to a pyproject.toml if it exists""" + path_project_root = find_project_root(path_search_start) + path_pyproject_toml = path_project_root / "pyproject.toml" + if path_pyproject_toml.is_file(): + return str(path_pyproject_toml) + + try: + path_user_pyproject_toml = find_user_pyproject_toml() + return ( + str(path_user_pyproject_toml) + if path_user_pyproject_toml.is_file() + else None + ) + except PermissionError as e: + # We do not have access to the user-level config directory, so ignore it. + err(f"Ignoring user configuration directory due to {e!r}") + return None + + +def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: + """Parse a pyproject toml file, pulling out relevant parts for Black + + If parsing fails, will raise a toml.TomlDecodeError + """ + pyproject_toml = toml.load(path_config) + config = pyproject_toml.get("tool", {}).get("black", {}) + return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()} + + +@lru_cache() +def find_user_pyproject_toml() -> Path: + r"""Return the path to the top-level user configuration for black. + + This looks for ~\.black on Windows and ~/.config/black on Linux and other + Unix systems. + """ + if sys.platform == "win32": + # Windows + user_config_path = Path.home() / ".black" + else: + config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config") + user_config_path = Path(config_root).expanduser() / "black" + return user_config_path.resolve() + + +@lru_cache() +def get_gitignore(root: Path) -> PathSpec: + """Return a PathSpec matching gitignore content if present.""" + gitignore = root / ".gitignore" + lines: List[str] = [] + if gitignore.is_file(): + with gitignore.open() as gf: + lines = gf.readlines() + return PathSpec.from_lines("gitwildmatch", lines) + + +def normalize_path_maybe_ignore( + path: Path, root: Path, report: Report +) -> Optional[str]: + """Normalize `path`. May return `None` if `path` was ignored. + + `report` is where "path ignored" output goes. + """ + try: + abspath = path if path.is_absolute() else Path.cwd() / path + normalized_path = abspath.resolve().relative_to(root).as_posix() + except OSError as e: + report.path_ignored(path, f"cannot be read because {e}") + return None + + except ValueError: + if path.is_symlink(): + report.path_ignored(path, f"is a symbolic link that points outside {root}") + return None + + raise + + return normalized_path + + +def path_is_excluded( + normalized_path: str, + pattern: Optional[Pattern[str]], +) -> bool: + match = pattern.search(normalized_path) if pattern else None + return bool(match and match.group(0)) + + +def gen_python_files( + paths: Iterable[Path], + root: Path, + include: Pattern[str], + exclude: Pattern[str], + extend_exclude: Optional[Pattern[str]], + force_exclude: Optional[Pattern[str]], + report: Report, + gitignore: Optional[PathSpec], +) -> Iterator[Path]: + """Generate all files under `path` whose paths are not excluded by the + `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, + but are included by the `include` regex. + + Symbolic links pointing outside of the `root` directory are ignored. + + `report` is where output about exclusions goes. + """ + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" + for child in paths: + normalized_path = normalize_path_maybe_ignore(child, root, report) + if normalized_path is None: + continue + + # First ignore files matching .gitignore, if passed + if gitignore is not None and gitignore.match_file(normalized_path): + report.path_ignored(child, "matches the .gitignore file content") + continue + + # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. + normalized_path = "/" + normalized_path + if child.is_dir(): + normalized_path += "/" + + if path_is_excluded(normalized_path, exclude): + report.path_ignored(child, "matches the --exclude regular expression") + continue + + if path_is_excluded(normalized_path, extend_exclude): + report.path_ignored( + child, "matches the --extend-exclude regular expression" + ) + continue + + if path_is_excluded(normalized_path, force_exclude): + report.path_ignored(child, "matches the --force-exclude regular expression") + continue + + if child.is_dir(): + yield from gen_python_files( + child.iterdir(), + root, + include, + exclude, + extend_exclude, + force_exclude, + report, + gitignore, + ) + + elif child.is_file(): + include_match = include.search(normalized_path) if include else True + if include_match: + yield child + + +def wrap_stream_for_windows( + f: io.TextIOWrapper, +) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]: + """ + Wrap stream with colorama's wrap_stream so colors are shown on Windows. + + If `colorama` is unavailable, the original stream is returned unmodified. + Otherwise, the `wrap_stream()` function determines whether the stream needs + to be wrapped for a Windows environment and will accordingly either return + an `AnsiToWin32` wrapper or the original stream. + """ + try: + from colorama.initialise import wrap_stream + except ImportError: + return f + else: + # Set `strip=False` to avoid needing to modify test_express_diff_with_color. + return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True) diff --git a/src/black/linegen.py b/src/black/linegen.py new file mode 100644 index 00000000000..2e16b6fde44 --- /dev/null +++ b/src/black/linegen.py @@ -0,0 +1,984 @@ +""" +Generating lines of code. +""" +from functools import partial, wraps +import sys +from typing import Collection, Iterator, List, Optional, Set, Union + +from dataclasses import dataclass, field + +from black.nodes import WHITESPACE, STATEMENT, STANDALONE_COMMENT +from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS +from black.nodes import Visitor, syms, first_child_is_arith, ensure_visible +from black.nodes import is_docstring, is_empty_tuple, is_one_tuple, is_one_tuple_between +from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string +from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens +from black.nodes import wrap_in_parentheses +from black.brackets import max_delimiter_priority_in_atom +from black.brackets import DOT_PRIORITY, COMMA_PRIORITY +from black.lines import Line, line_to_string, is_line_short_enough +from black.lines import can_omit_invisible_parens, can_be_split, append_leaves +from black.comments import generate_comments, list_comments, FMT_OFF +from black.numerics import normalize_numeric_literal +from black.strings import get_string_prefix, fix_docstring +from black.strings import normalize_string_prefix, normalize_string_quotes +from black.trans import Transformer, CannotTransform, StringMerger +from black.trans import StringSplitter, StringParenWrapper, StringParenStripper +from black.mode import Mode +from black.mode import Feature + +from blib2to3.pytree import Node, Leaf +from blib2to3.pgen2 import token + + +# types +LeafID = int +LN = Union[Leaf, Node] + + +class CannotSplit(CannotTransform): + """A readable split that fits the allotted line length is impossible.""" + + +@dataclass +class LineGenerator(Visitor[Line]): + """Generates reformatted Line objects. Empty lines are not emitted. + + Note: destroys the tree it's visiting by mutating prefixes of its leaves + in ways that will no longer stringify to valid Python code on the tree. + """ + + mode: Mode + remove_u_prefix: bool = False + current_line: Line = field(init=False) + + def line(self, indent: int = 0) -> Iterator[Line]: + """Generate a line. + + If the line is empty, only emit if it makes sense. + If the line is too long, split it first and then generate. + + If any lines were generated, set up a new current_line. + """ + if not self.current_line: + self.current_line.depth += indent + return # Line is empty, don't emit. Creating a new one unnecessary. + + complete_line = self.current_line + self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent) + yield complete_line + + def visit_default(self, node: LN) -> Iterator[Line]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" + if isinstance(node, Leaf): + any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() + for comment in generate_comments(node): + if any_open_brackets: + # any comment within brackets is subject to splitting + self.current_line.append(comment) + elif comment.type == token.COMMENT: + # regular trailing comment + self.current_line.append(comment) + yield from self.line() + + else: + # regular standalone comment + yield from self.line() + + self.current_line.append(comment) + yield from self.line() + + normalize_prefix(node, inside_brackets=any_open_brackets) + if self.mode.string_normalization and node.type == token.STRING: + node.value = normalize_string_prefix( + node.value, remove_u_prefix=self.remove_u_prefix + ) + node.value = normalize_string_quotes(node.value) + if node.type == token.NUMBER: + normalize_numeric_literal(node) + if node.type not in WHITESPACE: + self.current_line.append(node) + yield from super().visit_default(node) + + def visit_INDENT(self, node: Leaf) -> Iterator[Line]: + """Increase indentation level, maybe yield a line.""" + # In blib2to3 INDENT never holds comments. + yield from self.line(+1) + yield from self.visit_default(node) + + def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: + """Decrease indentation level, maybe yield a line.""" + # The current line might still wait for trailing comments. At DEDENT time + # there won't be any (they would be prefixes on the preceding NEWLINE). + # Emit the line then. + yield from self.line() + + # While DEDENT has no value, its prefix may contain standalone comments + # that belong to the current indentation level. Get 'em. + yield from self.visit_default(node) + + # Finally, emit the dedent. + yield from self.line(-1) + + def visit_stmt( + self, node: Node, keywords: Set[str], parens: Set[str] + ) -> Iterator[Line]: + """Visit a statement. + + This implementation is shared for `if`, `while`, `for`, `try`, `except`, + `def`, `with`, `class`, `assert` and assignments. + + The relevant Python language `keywords` for a given statement will be + NAME leaves within it. This methods puts those on a separate line. + + `parens` holds a set of string leaf values immediately after which + invisible parens should be put. + """ + normalize_invisible_parens(node, parens_after=parens) + for child in node.children: + if child.type == token.NAME and child.value in keywords: # type: ignore + yield from self.line() + + yield from self.visit(child) + + def visit_suite(self, node: Node) -> Iterator[Line]: + """Visit a suite.""" + if self.mode.is_pyi and is_stub_suite(node): + yield from self.visit(node.children[2]) + else: + yield from self.visit_default(node) + + def visit_simple_stmt(self, node: Node) -> Iterator[Line]: + """Visit a statement without nested statements.""" + if first_child_is_arith(node): + wrap_in_parentheses(node, node.children[0], visible=False) + is_suite_like = node.parent and node.parent.type in STATEMENT + if is_suite_like: + if self.mode.is_pyi and is_stub_body(node): + yield from self.visit_default(node) + else: + yield from self.line(+1) + yield from self.visit_default(node) + yield from self.line(-1) + + else: + if ( + not self.mode.is_pyi + or not node.parent + or not is_stub_suite(node.parent) + ): + yield from self.line() + yield from self.visit_default(node) + + def visit_async_stmt(self, node: Node) -> Iterator[Line]: + """Visit `async def`, `async for`, `async with`.""" + yield from self.line() + + children = iter(node.children) + for child in children: + yield from self.visit(child) + + if child.type == token.ASYNC: + break + + internal_stmt = next(children) + for child in internal_stmt.children: + yield from self.visit(child) + + def visit_decorators(self, node: Node) -> Iterator[Line]: + """Visit decorators.""" + for child in node.children: + yield from self.line() + yield from self.visit(child) + + def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: + """Remove a semicolon and put the other statement on a separate line.""" + yield from self.line() + + def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: + """End of file. Process outstanding comments and end with a newline.""" + yield from self.visit_default(leaf) + yield from self.line() + + def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: + if not self.current_line.bracket_tracker.any_open_brackets(): + yield from self.line() + yield from self.visit_default(leaf) + + def visit_factor(self, node: Node) -> Iterator[Line]: + """Force parentheses between a unary op and a binary power: + + -2 ** 8 -> -(2 ** 8) + """ + _operator, operand = node.children + if ( + operand.type == syms.power + and len(operand.children) == 3 + and operand.children[1].type == token.DOUBLESTAR + ): + lpar = Leaf(token.LPAR, "(") + rpar = Leaf(token.RPAR, ")") + index = operand.remove() or 0 + node.insert_child(index, Node(syms.atom, [lpar, operand, rpar])) + yield from self.visit_default(node) + + def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: + if is_docstring(leaf) and "\\\n" not in leaf.value: + # We're ignoring docstrings with backslash newline escapes because changing + # indentation of those changes the AST representation of the code. + prefix = get_string_prefix(leaf.value) + docstring = leaf.value[len(prefix) :] # Remove the prefix + quote_char = docstring[0] + # A natural way to remove the outer quotes is to do: + # docstring = docstring.strip(quote_char) + # but that breaks on """""x""" (which is '""x'). + # So we actually need to remove the first character and the next two + # characters but only if they are the same as the first. + quote_len = 1 if docstring[1] != quote_char else 3 + docstring = docstring[quote_len:-quote_len] + + if is_multiline_string(leaf): + indent = " " * 4 * self.current_line.depth + docstring = fix_docstring(docstring, indent) + else: + docstring = docstring.strip() + + if docstring: + # Add some padding if the docstring starts / ends with a quote mark. + if docstring[0] == quote_char: + docstring = " " + docstring + if docstring[-1] == quote_char: + docstring += " " + if docstring[-1] == "\\": + backslash_count = len(docstring) - len(docstring.rstrip("\\")) + if backslash_count % 2: + # Odd number of tailing backslashes, add some padding to + # avoid escaping the closing string quote. + docstring += " " + else: + # Add some padding if the docstring is empty. + docstring = " " + + # We could enforce triple quotes at this point. + quote = quote_char * quote_len + leaf.value = prefix + quote + docstring + quote + + yield from self.visit_default(leaf) + + def __post_init__(self) -> None: + """You are in a twisty little maze of passages.""" + self.current_line = Line(mode=self.mode) + + v = self.visit_stmt + Ø: Set[str] = set() + self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","}) + self.visit_if_stmt = partial( + v, keywords={"if", "else", "elif"}, parens={"if", "elif"} + ) + self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"}) + self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"}) + self.visit_try_stmt = partial( + v, keywords={"try", "except", "else", "finally"}, parens=Ø + ) + self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø) + self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø) + self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø) + self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) + self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) + self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) + self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) + self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) + self.visit_async_funcdef = self.visit_async_stmt + self.visit_decorated = self.visit_decorators + + +def transform_line( + line: Line, mode: Mode, features: Collection[Feature] = () +) -> Iterator[Line]: + """Transform a `line`, potentially splitting it into many lines. + + They should fit in the allotted `line_length` but might not be able to. + + `features` are syntactical features that may be used in the output. + """ + if line.is_comment: + yield line + return + + line_str = line_to_string(line) + + ll = mode.line_length + sn = mode.string_normalization + string_merge = StringMerger(ll, sn) + string_paren_strip = StringParenStripper(ll, sn) + string_split = StringSplitter(ll, sn) + string_paren_wrap = StringParenWrapper(ll, sn) + + transformers: List[Transformer] + if ( + not line.contains_uncollapsable_type_comments() + and not line.should_split_rhs + and not line.magic_trailing_comma + and ( + is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) + or line.contains_unsplittable_type_ignore() + ) + and not (line.inside_brackets and line.contains_standalone_comments()) + ): + # Only apply basic string preprocessing, since lines shouldn't be split here. + if mode.experimental_string_processing: + transformers = [string_merge, string_paren_strip] + else: + transformers = [] + elif line.is_def: + transformers = [left_hand_split] + else: + + def rhs(line: Line, features: Collection[Feature]) -> Iterator[Line]: + """Wraps calls to `right_hand_split`. + + The calls increasingly `omit` right-hand trailers (bracket pairs with + content), meaning the trailers get glued together to split on another + bracket pair instead. + """ + for omit in generate_trailers_to_omit(line, mode.line_length): + lines = list( + right_hand_split(line, mode.line_length, features, omit=omit) + ) + # Note: this check is only able to figure out if the first line of the + # *current* transformation fits in the line length. This is true only + # for simple cases. All others require running more transforms via + # `transform_line()`. This check doesn't know if those would succeed. + if is_line_short_enough(lines[0], line_length=mode.line_length): + yield from lines + return + + # All splits failed, best effort split with no omits. + # This mostly happens to multiline strings that are by definition + # reported as not fitting a single line, as well as lines that contain + # trailing commas (those have to be exploded). + yield from right_hand_split( + line, line_length=mode.line_length, features=features + ) + + if mode.experimental_string_processing: + if line.inside_brackets: + transformers = [ + string_merge, + string_paren_strip, + string_split, + delimiter_split, + standalone_comment_split, + string_paren_wrap, + rhs, + ] + else: + transformers = [ + string_merge, + string_paren_strip, + string_split, + string_paren_wrap, + rhs, + ] + else: + if line.inside_brackets: + transformers = [delimiter_split, standalone_comment_split, rhs] + else: + transformers = [rhs] + + for transform in transformers: + # We are accumulating lines in `result` because we might want to abort + # mission and return the original line in the end, or attempt a different + # split altogether. + try: + result = run_transformer(line, transform, mode, features, line_str=line_str) + except CannotTransform: + continue + else: + yield from result + break + + else: + yield line + + +def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]: + """Split line into many lines, starting with the first matching bracket pair. + + Note: this usually looks weird, only use this for function definitions. + Prefer RHS otherwise. This is why this function is not symmetrical with + :func:`right_hand_split` which also handles optional parentheses. + """ + tail_leaves: List[Leaf] = [] + body_leaves: List[Leaf] = [] + head_leaves: List[Leaf] = [] + current_leaves = head_leaves + matching_bracket: Optional[Leaf] = None + for leaf in line.leaves: + if ( + current_leaves is body_leaves + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket is matching_bracket + ): + current_leaves = tail_leaves if body_leaves else head_leaves + current_leaves.append(leaf) + if current_leaves is head_leaves: + if leaf.type in OPENING_BRACKETS: + matching_bracket = leaf + current_leaves = body_leaves + if not matching_bracket: + raise CannotSplit("No brackets found") + + head = bracket_split_build_line(head_leaves, line, matching_bracket) + body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True) + tail = bracket_split_build_line(tail_leaves, line, matching_bracket) + bracket_split_succeeded_or_raise(head, body, tail) + for result in (head, body, tail): + if result: + yield result + + +def right_hand_split( + line: Line, + line_length: int, + features: Collection[Feature] = (), + omit: Collection[LeafID] = (), +) -> Iterator[Line]: + """Split line into many lines, starting with the last matching bracket pair. + + If the split was by optional parentheses, attempt splitting without them, too. + `omit` is a collection of closing bracket IDs that shouldn't be considered for + this split. + + Note: running this function modifies `bracket_depth` on the leaves of `line`. + """ + tail_leaves: List[Leaf] = [] + body_leaves: List[Leaf] = [] + head_leaves: List[Leaf] = [] + current_leaves = tail_leaves + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if current_leaves is body_leaves: + if leaf is opening_bracket: + current_leaves = head_leaves if body_leaves else tail_leaves + current_leaves.append(leaf) + if current_leaves is tail_leaves: + if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit: + opening_bracket = leaf.opening_bracket + closing_bracket = leaf + current_leaves = body_leaves + if not (opening_bracket and closing_bracket and head_leaves): + # If there is no opening or closing_bracket that means the split failed and + # all content is in the tail. Otherwise, if `head_leaves` are empty, it means + # the matching `opening_bracket` wasn't available on `line` anymore. + raise CannotSplit("No brackets found") + + tail_leaves.reverse() + body_leaves.reverse() + head_leaves.reverse() + head = bracket_split_build_line(head_leaves, line, opening_bracket) + body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True) + tail = bracket_split_build_line(tail_leaves, line, opening_bracket) + bracket_split_succeeded_or_raise(head, body, tail) + if ( + Feature.FORCE_OPTIONAL_PARENTHESES not in features + # the opening bracket is an optional paren + and opening_bracket.type == token.LPAR + and not opening_bracket.value + # the closing bracket is an optional paren + and closing_bracket.type == token.RPAR + and not closing_bracket.value + # it's not an import (optional parens are the only thing we can split on + # in this case; attempting a split without them is a waste of time) + and not line.is_import + # there are no standalone comments in the body + and not body.contains_standalone_comments(0) + # and we can actually remove the parens + and can_omit_invisible_parens(body, line_length, omit_on_explode=omit) + ): + omit = {id(closing_bracket), *omit} + try: + yield from right_hand_split(line, line_length, features=features, omit=omit) + return + + except CannotSplit: + if not ( + can_be_split(body) + or is_line_short_enough(body, line_length=line_length) + ): + raise CannotSplit( + "Splitting failed, body is still too long and can't be split." + ) + + elif head.contains_multiline_strings() or tail.contains_multiline_strings(): + raise CannotSplit( + "The current optional pair of parentheses is bound to fail to" + " satisfy the splitting algorithm because the head or the tail" + " contains multiline strings which by definition never fit one" + " line." + ) + + ensure_visible(opening_bracket) + ensure_visible(closing_bracket) + for result in (head, body, tail): + if result: + yield result + + +def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: + """Raise :exc:`CannotSplit` if the last left- or right-hand split failed. + + Do nothing otherwise. + + A left- or right-hand split is based on a pair of brackets. Content before + (and including) the opening bracket is left on one line, content inside the + brackets is put on a separate line, and finally content starting with and + following the closing bracket is put on a separate line. + + Those are called `head`, `body`, and `tail`, respectively. If the split + produced the same line (all content in `head`) or ended up with an empty `body` + and the `tail` is just the closing bracket, then it's considered failed. + """ + tail_len = len(str(tail).strip()) + if not body: + if tail_len == 0: + raise CannotSplit("Splitting brackets produced the same line") + + elif tail_len < 3: + raise CannotSplit( + f"Splitting brackets on an empty body to save {tail_len} characters is" + " not worth it" + ) + + +def bracket_split_build_line( + leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False +) -> Line: + """Return a new line with given `leaves` and respective comments from `original`. + + If `is_body` is True, the result line is one-indented inside brackets and as such + has its first leaf's prefix normalized and a trailing comma added when expected. + """ + result = Line(mode=original.mode, depth=original.depth) + if is_body: + result.inside_brackets = True + result.depth += 1 + if leaves: + # Since body is a new indent level, remove spurious leading whitespace. + normalize_prefix(leaves[0], inside_brackets=True) + # Ensure a trailing comma for imports and standalone function arguments, but + # be careful not to add one after any comments or within type annotations. + no_commas = ( + original.is_def + and opening_bracket.value == "(" + and not any(leaf.type == token.COMMA for leaf in leaves) + ) + + if original.is_import or no_commas: + for i in range(len(leaves) - 1, -1, -1): + if leaves[i].type == STANDALONE_COMMENT: + continue + + if leaves[i].type != token.COMMA: + new_comma = Leaf(token.COMMA, ",") + leaves.insert(i + 1, new_comma) + break + + # Populate the line + for leaf in leaves: + result.append(leaf, preformatted=True) + for comment_after in original.comments_after(leaf): + result.append(comment_after, preformatted=True) + if is_body and should_split_line(result, opening_bracket): + result.should_split_rhs = True + return result + + +def dont_increase_indentation(split_func: Transformer) -> Transformer: + """Normalize prefix of the first leaf in every line returned by `split_func`. + + This is a decorator over relevant split functions. + """ + + @wraps(split_func) + def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: + for line in split_func(line, features): + normalize_prefix(line.leaves[0], inside_brackets=True) + yield line + + return split_wrapper + + +@dont_increase_indentation +def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: + """Split according to delimiters of the highest priority. + + If the appropriate Features are given, the split will add trailing commas + also in function signatures and calls that contain `*` and `**`. + """ + try: + last_leaf = line.leaves[-1] + except IndexError: + raise CannotSplit("Line empty") + + bt = line.bracket_tracker + try: + delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)}) + except ValueError: + raise CannotSplit("No delimiters found") + + if delimiter_priority == DOT_PRIORITY: + if bt.delimiter_count_with_priority(delimiter_priority) == 1: + raise CannotSplit("Splitting a single attribute from its owner looks wrong") + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + lowest_depth = sys.maxsize + trailing_comma_safe = True + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError: + yield current_line + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + current_line.append(leaf) + + for leaf in line.leaves: + yield from append_to_line(leaf) + + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + + lowest_depth = min(lowest_depth, leaf.bracket_depth) + if leaf.bracket_depth == lowest_depth: + if is_vararg(leaf, within={syms.typedargslist}): + trailing_comma_safe = ( + trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features + ) + elif is_vararg(leaf, within={syms.arglist, syms.argument}): + trailing_comma_safe = ( + trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features + ) + + leaf_priority = bt.delimiters.get(id(leaf)) + if leaf_priority == delimiter_priority: + yield current_line + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + if current_line: + if ( + trailing_comma_safe + and delimiter_priority == COMMA_PRIORITY + and current_line.leaves[-1].type != token.COMMA + and current_line.leaves[-1].type != STANDALONE_COMMENT + ): + new_comma = Leaf(token.COMMA, ",") + current_line.append(new_comma) + yield current_line + + +@dont_increase_indentation +def standalone_comment_split( + line: Line, features: Collection[Feature] = () +) -> Iterator[Line]: + """Split standalone comments from the rest of the line.""" + if not line.contains_standalone_comments(0): + raise CannotSplit("Line does not have any standalone comments") + + current_line = Line( + mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + + def append_to_line(leaf: Leaf) -> Iterator[Line]: + """Append `leaf` to current line or to new line if appending impossible.""" + nonlocal current_line + try: + current_line.append_safe(leaf, preformatted=True) + except ValueError: + yield current_line + + current_line = Line( + line.mode, depth=line.depth, inside_brackets=line.inside_brackets + ) + current_line.append(leaf) + + for leaf in line.leaves: + yield from append_to_line(leaf) + + for comment_after in line.comments_after(leaf): + yield from append_to_line(comment_after) + + if current_line: + yield current_line + + +def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: + """Leave existing extra newlines if not `inside_brackets`. Remove everything + else. + + Note: don't use backslashes for formatting or you'll lose your voting rights. + """ + if not inside_brackets: + spl = leaf.prefix.split("#") + if "\\" not in spl[0]: + nl_count = spl[-1].count("\n") + if len(spl) > 1: + nl_count -= 1 + leaf.prefix = "\n" * nl_count + return + + leaf.prefix = "" + + +def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None: + """Make existing optional parentheses invisible or create new ones. + + `parens_after` is a set of string leaf values immediately after which parens + should be put. + + Standardizes on visible parentheses for single-element tuples, and keeps + existing visible parentheses for other tuples and generator expressions. + """ + for pc in list_comments(node.prefix, is_endmarker=False): + if pc.value in FMT_OFF: + # This `node` has a prefix with `# fmt: off`, don't mess with parens. + return + check_lpar = False + for index, child in enumerate(list(node.children)): + # Fixes a bug where invisible parens are not properly stripped from + # assignment statements that contain type annotations. + if isinstance(child, Node) and child.type == syms.annassign: + normalize_invisible_parens(child, parens_after=parens_after) + + # Add parentheses around long tuple unpacking in assignments. + if ( + index == 0 + and isinstance(child, Node) + and child.type == syms.testlist_star_expr + ): + check_lpar = True + + if check_lpar: + if child.type == syms.atom: + if maybe_make_parens_invisible_in_atom(child, parent=node): + wrap_in_parentheses(node, child, visible=False) + elif is_one_tuple(child): + wrap_in_parentheses(node, child, visible=True) + elif node.type == syms.import_from: + # "import from" nodes store parentheses directly as part of + # the statement + if child.type == token.LPAR: + # make parentheses invisible + child.value = "" # type: ignore + node.children[-1].value = "" # type: ignore + elif child.type != token.STAR: + # insert invisible parentheses + node.insert_child(index, Leaf(token.LPAR, "")) + node.append_child(Leaf(token.RPAR, "")) + break + + elif not (isinstance(child, Leaf) and is_multiline_string(child)): + wrap_in_parentheses(node, child, visible=False) + + check_lpar = isinstance(child, Leaf) and child.value in parens_after + + +def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: + """If it's safe, make the parens in the atom `node` invisible, recursively. + Additionally, remove repeated, adjacent invisible parens from the atom `node` + as they are redundant. + + Returns whether the node should itself be wrapped in invisible parentheses. + + """ + + if ( + node.type != syms.atom + or is_empty_tuple(node) + or is_one_tuple(node) + or (is_yield(node) and parent.type != syms.expr_stmt) + or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY + ): + return False + + if is_walrus_assignment(node): + if parent.type in [ + syms.annassign, + syms.expr_stmt, + syms.assert_stmt, + syms.return_stmt, + # these ones aren't useful to end users, but they do please fuzzers + syms.for_stmt, + syms.del_stmt, + ]: + return False + + first = node.children[0] + last = node.children[-1] + if first.type == token.LPAR and last.type == token.RPAR: + middle = node.children[1] + # make parentheses invisible + first.value = "" # type: ignore + last.value = "" # type: ignore + maybe_make_parens_invisible_in_atom(middle, parent=parent) + + if is_atom_with_invisible_parens(middle): + # Strip the invisible parens from `middle` by replacing + # it with the child in-between the invisible parens + middle.replace(middle.children[1]) + + return False + + return True + + +def should_split_line(line: Line, opening_bracket: Leaf) -> bool: + """Should `line` be immediately split with `delimiter_split()` after RHS?""" + + if not (opening_bracket.parent and opening_bracket.value in "[{("): + return False + + # We're essentially checking if the body is delimited by commas and there's more + # than one of them (we're excluding the trailing comma and if the delimiter priority + # is still commas, that means there's more). + exclude = set() + trailing_comma = False + try: + last_leaf = line.leaves[-1] + if last_leaf.type == token.COMMA: + trailing_comma = True + exclude.add(id(last_leaf)) + max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude) + except (IndexError, ValueError): + return False + + return max_priority == COMMA_PRIORITY and ( + (line.mode.magic_trailing_comma and trailing_comma) + # always explode imports + or opening_bracket.parent.type in {syms.atom, syms.import_from} + ) + + +def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]: + """Generate sets of closing bracket IDs that should be omitted in a RHS. + + Brackets can be omitted if the entire trailer up to and including + a preceding closing bracket fits in one line. + + Yielded sets are cumulative (contain results of previous yields, too). First + set is empty, unless the line should explode, in which case bracket pairs until + the one that needs to explode are omitted. + """ + + omit: Set[LeafID] = set() + if not line.magic_trailing_comma: + yield omit + + length = 4 * line.depth + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None + inner_brackets: Set[LeafID] = set() + for index, leaf, leaf_length in line.enumerate_with_length(reversed=True): + length += leaf_length + if length > line_length: + break + + has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix) + if leaf.type == STANDALONE_COMMENT or has_inline_comment: + break + + if opening_bracket: + if leaf is opening_bracket: + opening_bracket = None + elif leaf.type in CLOSING_BRACKETS: + prev = line.leaves[index - 1] if index > 0 else None + if ( + prev + and prev.type == token.COMMA + and not is_one_tuple_between( + leaf.opening_bracket, leaf, line.leaves + ) + ): + # Never omit bracket pairs with trailing commas. + # We need to explode on those. + break + + inner_brackets.add(id(leaf)) + elif leaf.type in CLOSING_BRACKETS: + prev = line.leaves[index - 1] if index > 0 else None + if prev and prev.type in OPENING_BRACKETS: + # Empty brackets would fail a split so treat them as "inner" + # brackets (e.g. only add them to the `omit` set if another + # pair of brackets was good enough. + inner_brackets.add(id(leaf)) + continue + + if closing_bracket: + omit.add(id(closing_bracket)) + omit.update(inner_brackets) + inner_brackets.clear() + yield omit + + if ( + prev + and prev.type == token.COMMA + and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves) + ): + # Never omit bracket pairs with trailing commas. + # We need to explode on those. + break + + if leaf.value: + opening_bracket = leaf.opening_bracket + closing_bracket = leaf + + +def run_transformer( + line: Line, + transform: Transformer, + mode: Mode, + features: Collection[Feature], + *, + line_str: str = "", +) -> List[Line]: + if not line_str: + line_str = line_to_string(line) + result: List[Line] = [] + for transformed_line in transform(line, features): + if str(transformed_line).strip("\n") == line_str: + raise CannotTransform("Line transformer returned an unchanged result") + + result.extend(transform_line(transformed_line, mode=mode, features=features)) + + if not ( + transform.__name__ == "rhs" + and line.bracket_tracker.invisible + and not any(bracket.value for bracket in line.bracket_tracker.invisible) + and not line.contains_multiline_strings() + and not result[0].contains_uncollapsable_type_comments() + and not result[0].contains_unsplittable_type_ignore() + and not is_line_short_enough(result[0], line_length=mode.line_length) + ): + return result + + line_copy = line.clone() + append_leaves(line_copy, line, line.leaves) + features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES} + second_opinion = run_transformer( + line_copy, transform, mode, features_fop, line_str=line_str + ) + if all( + is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion + ): + result = second_opinion + return result diff --git a/src/black/lines.py b/src/black/lines.py new file mode 100644 index 00000000000..63225c0e6d3 --- /dev/null +++ b/src/black/lines.py @@ -0,0 +1,734 @@ +from dataclasses import dataclass, field +import itertools +import sys +from typing import ( + Callable, + Collection, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypeVar, + cast, +) + +from blib2to3.pytree import Node, Leaf +from blib2to3.pgen2 import token + +from black.brackets import BracketTracker, DOT_PRIORITY +from black.mode import Mode +from black.nodes import STANDALONE_COMMENT, TEST_DESCENDANTS +from black.nodes import BRACKETS, OPENING_BRACKETS, CLOSING_BRACKETS +from black.nodes import syms, whitespace, replace_child, child_towards +from black.nodes import is_multiline_string, is_import, is_type_comment, last_two_except +from black.nodes import is_one_tuple_between + +# types +T = TypeVar("T") +Index = int +LeafID = int + + +@dataclass +class Line: + """Holds leaves and comments. Can be printed with `str(line)`.""" + + mode: Mode + depth: int = 0 + leaves: List[Leaf] = field(default_factory=list) + # keys ordered like `leaves` + comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) + bracket_tracker: BracketTracker = field(default_factory=BracketTracker) + inside_brackets: bool = False + should_split_rhs: bool = False + magic_trailing_comma: Optional[Leaf] = None + + def append(self, leaf: Leaf, preformatted: bool = False) -> None: + """Add a new `leaf` to the end of the line. + + Unless `preformatted` is True, the `leaf` will receive a new consistent + whitespace prefix and metadata applied by :class:`BracketTracker`. + Trailing commas are maybe removed, unpacked for loop variables are + demoted from being delimiters. + + Inline comments are put aside. + """ + has_value = leaf.type in BRACKETS or bool(leaf.value.strip()) + if not has_value: + return + + if token.COLON == leaf.type and self.is_class_paren_empty: + del self.leaves[-2:] + if self.leaves and not preformatted: + # Note: at this point leaf.prefix should be empty except for + # imports, for which we only preserve newlines. + leaf.prefix += whitespace( + leaf, complex_subscript=self.is_complex_subscript(leaf) + ) + if self.inside_brackets or not preformatted: + self.bracket_tracker.mark(leaf) + if self.mode.magic_trailing_comma: + if self.has_magic_trailing_comma(leaf): + self.magic_trailing_comma = leaf + elif self.has_magic_trailing_comma(leaf, ensure_removable=True): + self.remove_trailing_comma() + if not self.append_comment(leaf): + self.leaves.append(leaf) + + def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: + """Like :func:`append()` but disallow invalid standalone comment structure. + + Raises ValueError when any `leaf` is appended after a standalone comment + or when a standalone comment is not the first leaf on the line. + """ + if self.bracket_tracker.depth == 0: + if self.is_comment: + raise ValueError("cannot append to standalone comments") + + if self.leaves and leaf.type == STANDALONE_COMMENT: + raise ValueError( + "cannot append standalone comments to a populated line" + ) + + self.append(leaf, preformatted=preformatted) + + @property + def is_comment(self) -> bool: + """Is this line a standalone comment?""" + return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT + + @property + def is_decorator(self) -> bool: + """Is this line a decorator?""" + return bool(self) and self.leaves[0].type == token.AT + + @property + def is_import(self) -> bool: + """Is this an import line?""" + return bool(self) and is_import(self.leaves[0]) + + @property + def is_class(self) -> bool: + """Is this line a class definition?""" + return ( + bool(self) + and self.leaves[0].type == token.NAME + and self.leaves[0].value == "class" + ) + + @property + def is_stub_class(self) -> bool: + """Is this line a class definition with a body consisting only of "..."?""" + return self.is_class and self.leaves[-3:] == [ + Leaf(token.DOT, ".") for _ in range(3) + ] + + @property + def is_def(self) -> bool: + """Is this a function definition? (Also returns True for async defs.)""" + try: + first_leaf = self.leaves[0] + except IndexError: + return False + + try: + second_leaf: Optional[Leaf] = self.leaves[1] + except IndexError: + second_leaf = None + return (first_leaf.type == token.NAME and first_leaf.value == "def") or ( + first_leaf.type == token.ASYNC + and second_leaf is not None + and second_leaf.type == token.NAME + and second_leaf.value == "def" + ) + + @property + def is_class_paren_empty(self) -> bool: + """Is this a class with no base classes but using parentheses? + + Those are unnecessary and should be removed. + """ + return ( + bool(self) + and len(self.leaves) == 4 + and self.is_class + and self.leaves[2].type == token.LPAR + and self.leaves[2].value == "(" + and self.leaves[3].type == token.RPAR + and self.leaves[3].value == ")" + ) + + @property + def is_triple_quoted_string(self) -> bool: + """Is the line a triple quoted string?""" + return ( + bool(self) + and self.leaves[0].type == token.STRING + and self.leaves[0].value.startswith(('"""', "'''")) + ) + + def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: + """If so, needs to be split before emitting.""" + for leaf in self.leaves: + if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit: + return True + + return False + + def contains_uncollapsable_type_comments(self) -> bool: + ignored_ids = set() + try: + last_leaf = self.leaves[-1] + ignored_ids.add(id(last_leaf)) + if last_leaf.type == token.COMMA or ( + last_leaf.type == token.RPAR and not last_leaf.value + ): + # When trailing commas or optional parens are inserted by Black for + # consistency, comments after the previous last element are not moved + # (they don't have to, rendering will still be correct). So we ignore + # trailing commas and invisible. + last_leaf = self.leaves[-2] + ignored_ids.add(id(last_leaf)) + except IndexError: + return False + + # A type comment is uncollapsable if it is attached to a leaf + # that isn't at the end of the line (since that could cause it + # to get associated to a different argument) or if there are + # comments before it (since that could cause it to get hidden + # behind a comment. + comment_seen = False + for leaf_id, comments in self.comments.items(): + for comment in comments: + if is_type_comment(comment): + if comment_seen or ( + not is_type_comment(comment, " ignore") + and leaf_id not in ignored_ids + ): + return True + + comment_seen = True + + return False + + def contains_unsplittable_type_ignore(self) -> bool: + if not self.leaves: + return False + + # If a 'type: ignore' is attached to the end of a line, we + # can't split the line, because we can't know which of the + # subexpressions the ignore was meant to apply to. + # + # We only want this to apply to actual physical lines from the + # original source, though: we don't want the presence of a + # 'type: ignore' at the end of a multiline expression to + # justify pushing it all onto one line. Thus we + # (unfortunately) need to check the actual source lines and + # only report an unsplittable 'type: ignore' if this line was + # one line in the original code. + + # Grab the first and last line numbers, skipping generated leaves + first_line = next((leaf.lineno for leaf in self.leaves if leaf.lineno != 0), 0) + last_line = next( + (leaf.lineno for leaf in reversed(self.leaves) if leaf.lineno != 0), 0 + ) + + if first_line == last_line: + # We look at the last two leaves since a comma or an + # invisible paren could have been added at the end of the + # line. + for node in self.leaves[-2:]: + for comment in self.comments.get(id(node), []): + if is_type_comment(comment, " ignore"): + return True + + return False + + def contains_multiline_strings(self) -> bool: + return any(is_multiline_string(leaf) for leaf in self.leaves) + + def has_magic_trailing_comma( + self, closing: Leaf, ensure_removable: bool = False + ) -> bool: + """Return True if we have a magic trailing comma, that is when: + - there's a trailing comma here + - it's not a one-tuple + Additionally, if ensure_removable: + - it's not from square bracket indexing + """ + if not ( + closing.type in CLOSING_BRACKETS + and self.leaves + and self.leaves[-1].type == token.COMMA + ): + return False + + if closing.type == token.RBRACE: + return True + + if closing.type == token.RSQB: + if not ensure_removable: + return True + comma = self.leaves[-1] + return bool(comma.parent and comma.parent.type == syms.listmaker) + + if self.is_import: + return True + + if not is_one_tuple_between(closing.opening_bracket, closing, self.leaves): + return True + + return False + + def append_comment(self, comment: Leaf) -> bool: + """Add an inline or standalone comment to the line.""" + if ( + comment.type == STANDALONE_COMMENT + and self.bracket_tracker.any_open_brackets() + ): + comment.prefix = "" + return False + + if comment.type != token.COMMENT: + return False + + if not self.leaves: + comment.type = STANDALONE_COMMENT + comment.prefix = "" + return False + + last_leaf = self.leaves[-1] + if ( + last_leaf.type == token.RPAR + and not last_leaf.value + and last_leaf.parent + and len(list(last_leaf.parent.leaves())) <= 3 + and not is_type_comment(comment) + ): + # Comments on an optional parens wrapping a single leaf should belong to + # the wrapped node except if it's a type comment. Pinning the comment like + # this avoids unstable formatting caused by comment migration. + if len(self.leaves) < 2: + comment.type = STANDALONE_COMMENT + comment.prefix = "" + return False + + last_leaf = self.leaves[-2] + self.comments.setdefault(id(last_leaf), []).append(comment) + return True + + def comments_after(self, leaf: Leaf) -> List[Leaf]: + """Generate comments that should appear directly after `leaf`.""" + return self.comments.get(id(leaf), []) + + def remove_trailing_comma(self) -> None: + """Remove the trailing comma and moves the comments attached to it.""" + trailing_comma = self.leaves.pop() + trailing_comma_comments = self.comments.pop(id(trailing_comma), []) + self.comments.setdefault(id(self.leaves[-1]), []).extend( + trailing_comma_comments + ) + + def is_complex_subscript(self, leaf: Leaf) -> bool: + """Return True iff `leaf` is part of a slice with non-trivial exprs.""" + open_lsqb = self.bracket_tracker.get_open_lsqb() + if open_lsqb is None: + return False + + subscript_start = open_lsqb.next_sibling + + if isinstance(subscript_start, Node): + if subscript_start.type == syms.listmaker: + return False + + if subscript_start.type == syms.subscriptlist: + subscript_start = child_towards(subscript_start, leaf) + return subscript_start is not None and any( + n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() + ) + + def enumerate_with_length( + self, reversed: bool = False + ) -> Iterator[Tuple[Index, Leaf, int]]: + """Return an enumeration of leaves with their length. + + Stops prematurely on multiline strings and standalone comments. + """ + op = cast( + Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]], + enumerate_reversed if reversed else enumerate, + ) + for index, leaf in op(self.leaves): + length = len(leaf.prefix) + len(leaf.value) + if "\n" in leaf.value: + return # Multiline strings, we can't continue. + + for comment in self.comments_after(leaf): + length += len(comment.value) + + yield index, leaf, length + + def clone(self) -> "Line": + return Line( + mode=self.mode, + depth=self.depth, + inside_brackets=self.inside_brackets, + should_split_rhs=self.should_split_rhs, + magic_trailing_comma=self.magic_trailing_comma, + ) + + def __str__(self) -> str: + """Render the line.""" + if not self: + return "\n" + + indent = " " * self.depth + leaves = iter(self.leaves) + first = next(leaves) + res = f"{first.prefix}{indent}{first.value}" + for leaf in leaves: + res += str(leaf) + for comment in itertools.chain.from_iterable(self.comments.values()): + res += str(comment) + + return res + "\n" + + def __bool__(self) -> bool: + """Return True if the line has leaves or comments.""" + return bool(self.leaves or self.comments) + + +@dataclass +class EmptyLineTracker: + """Provides a stateful method that returns the number of potential extra + empty lines needed before and after the currently processed line. + + Note: this tracker works on lines that haven't been split yet. It assumes + the prefix of the first leaf consists of optional newlines. Those newlines + are consumed by `maybe_empty_lines()` and included in the computation. + """ + + is_pyi: bool = False + previous_line: Optional[Line] = None + previous_after: int = 0 + previous_defs: List[int] = field(default_factory=list) + + def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: + """Return the number of extra empty lines before and after the `current_line`. + + This is for separating `def`, `async def` and `class` with extra empty + lines (two on module-level). + """ + before, after = self._maybe_empty_lines(current_line) + before = ( + # Black should not insert empty lines at the beginning + # of the file + 0 + if self.previous_line is None + else before - self.previous_after + ) + self.previous_after = after + self.previous_line = current_line + return before, after + + def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: + max_allowed = 1 + if current_line.depth == 0: + max_allowed = 1 if self.is_pyi else 2 + if current_line.leaves: + # Consume the first leaf's extra newlines. + first_leaf = current_line.leaves[0] + before = first_leaf.prefix.count("\n") + before = min(before, max_allowed) + first_leaf.prefix = "" + else: + before = 0 + depth = current_line.depth + while self.previous_defs and self.previous_defs[-1] >= depth: + self.previous_defs.pop() + if self.is_pyi: + before = 0 if depth else 1 + else: + before = 1 if depth else 2 + if current_line.is_decorator or current_line.is_def or current_line.is_class: + return self._maybe_empty_lines_for_class_or_def(current_line, before) + + if ( + self.previous_line + and self.previous_line.is_import + and not current_line.is_import + and depth == self.previous_line.depth + ): + return (before or 1), 0 + + if ( + self.previous_line + and self.previous_line.is_class + and current_line.is_triple_quoted_string + ): + return before, 1 + + return before, 0 + + def _maybe_empty_lines_for_class_or_def( + self, current_line: Line, before: int + ) -> Tuple[int, int]: + if not current_line.is_decorator: + self.previous_defs.append(current_line.depth) + if self.previous_line is None: + # Don't insert empty lines before the first line in the file. + return 0, 0 + + if self.previous_line.is_decorator: + if self.is_pyi and current_line.is_stub_class: + # Insert an empty line after a decorated stub class + return 0, 1 + + return 0, 0 + + if self.previous_line.depth < current_line.depth and ( + self.previous_line.is_class or self.previous_line.is_def + ): + return 0, 0 + + if ( + self.previous_line.is_comment + and self.previous_line.depth == current_line.depth + and before == 0 + ): + return 0, 0 + + if self.is_pyi: + if self.previous_line.depth > current_line.depth: + newlines = 1 + elif current_line.is_class or self.previous_line.is_class: + if current_line.is_stub_class and self.previous_line.is_stub_class: + # No blank line between classes with an empty body + newlines = 0 + else: + newlines = 1 + elif ( + current_line.is_def or current_line.is_decorator + ) and not self.previous_line.is_def: + # Blank line between a block of functions (maybe with preceding + # decorators) and a block of non-functions + newlines = 1 + else: + newlines = 0 + else: + newlines = 2 + if current_line.depth and newlines: + newlines -= 1 + return newlines, 0 + + +def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]: + """Like `reversed(enumerate(sequence))` if that were possible.""" + index = len(sequence) - 1 + for element in reversed(sequence): + yield (index, element) + index -= 1 + + +def append_leaves( + new_line: Line, old_line: Line, leaves: List[Leaf], preformatted: bool = False +) -> None: + """ + Append leaves (taken from @old_line) to @new_line, making sure to fix the + underlying Node structure where appropriate. + + All of the leaves in @leaves are duplicated. The duplicates are then + appended to @new_line and used to replace their originals in the underlying + Node structure. Any comments attached to the old leaves are reattached to + the new leaves. + + Pre-conditions: + set(@leaves) is a subset of set(@old_line.leaves). + """ + for old_leaf in leaves: + new_leaf = Leaf(old_leaf.type, old_leaf.value) + replace_child(old_leaf, new_leaf) + new_line.append(new_leaf, preformatted=preformatted) + + for comment_leaf in old_line.comments_after(old_leaf): + new_line.append(comment_leaf, preformatted=True) + + +def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool: + """Return True if `line` is no longer than `line_length`. + + Uses the provided `line_str` rendering, if any, otherwise computes a new one. + """ + if not line_str: + line_str = line_to_string(line) + return ( + len(line_str) <= line_length + and "\n" not in line_str # multiline strings + and not line.contains_standalone_comments() + ) + + +def can_be_split(line: Line) -> bool: + """Return False if the line cannot be split *for sure*. + + This is not an exhaustive search but a cheap heuristic that we can use to + avoid some unfortunate formattings (mostly around wrapping unsplittable code + in unnecessary parentheses). + """ + leaves = line.leaves + if len(leaves) < 2: + return False + + if leaves[0].type == token.STRING and leaves[1].type == token.DOT: + call_count = 0 + dot_count = 0 + next = leaves[-1] + for leaf in leaves[-2::-1]: + if leaf.type in OPENING_BRACKETS: + if next.type not in CLOSING_BRACKETS: + return False + + call_count += 1 + elif leaf.type == token.DOT: + dot_count += 1 + elif leaf.type == token.NAME: + if not (next.type == token.DOT or next.type in OPENING_BRACKETS): + return False + + elif leaf.type not in CLOSING_BRACKETS: + return False + + if dot_count > 1 and call_count > 1: + return False + + return True + + +def can_omit_invisible_parens( + line: Line, + line_length: int, + omit_on_explode: Collection[LeafID] = (), +) -> bool: + """Does `line` have a shape safe to reformat without optional parens around it? + + Returns True for only a subset of potentially nice looking formattings but + the point is to not return false positives that end up producing lines that + are too long. + """ + bt = line.bracket_tracker + if not bt.delimiters: + # Without delimiters the optional parentheses are useless. + return True + + max_priority = bt.max_delimiter_priority() + if bt.delimiter_count_with_priority(max_priority) > 1: + # With more than one delimiter of a kind the optional parentheses read better. + return False + + if max_priority == DOT_PRIORITY: + # A single stranded method call doesn't require optional parentheses. + return True + + assert len(line.leaves) >= 2, "Stranded delimiter" + + # With a single delimiter, omit if the expression starts or ends with + # a bracket. + first = line.leaves[0] + second = line.leaves[1] + if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS: + if _can_omit_opening_paren(line, first=first, line_length=line_length): + return True + + # Note: we are not returning False here because a line might have *both* + # a leading opening bracket and a trailing closing bracket. If the + # opening bracket doesn't match our rule, maybe the closing will. + + penultimate = line.leaves[-2] + last = line.leaves[-1] + if line.magic_trailing_comma: + try: + penultimate, last = last_two_except(line.leaves, omit=omit_on_explode) + except LookupError: + # Turns out we'd omit everything. We cannot skip the optional parentheses. + return False + + if ( + last.type == token.RPAR + or last.type == token.RBRACE + or ( + # don't use indexing for omitting optional parentheses; + # it looks weird + last.type == token.RSQB + and last.parent + and last.parent.type != syms.trailer + ) + ): + if penultimate.type in OPENING_BRACKETS: + # Empty brackets don't help. + return False + + if is_multiline_string(first): + # Additional wrapping of a multiline string in this situation is + # unnecessary. + return True + + if line.magic_trailing_comma and penultimate.type == token.COMMA: + # The rightmost non-omitted bracket pair is the one we want to explode on. + return True + + if _can_omit_closing_paren(line, last=last, line_length=line_length): + return True + + return False + + +def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: + """See `can_omit_invisible_parens`.""" + remainder = False + length = 4 * line.depth + _index = -1 + for _index, leaf, leaf_length in line.enumerate_with_length(): + if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: + remainder = True + if remainder: + length += leaf_length + if length > line_length: + break + + if leaf.type in OPENING_BRACKETS: + # There are brackets we can further split on. + remainder = False + + else: + # checked the entire string and line length wasn't exceeded + if len(line.leaves) == _index + 1: + return True + + return False + + +def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: + """See `can_omit_invisible_parens`.""" + length = 4 * line.depth + seen_other_brackets = False + for _index, leaf, leaf_length in line.enumerate_with_length(): + length += leaf_length + if leaf is last.opening_bracket: + if seen_other_brackets or length <= line_length: + return True + + elif leaf.type in OPENING_BRACKETS: + # There are brackets we can further split on. + seen_other_brackets = True + + return False + + +def line_to_string(line: Line) -> str: + """Returns the string representation of @line. + + WARNING: This is known to be computationally expensive. + """ + return str(line).strip("\n") diff --git a/src/black/mode.py b/src/black/mode.py new file mode 100644 index 00000000000..e2ce322da5c --- /dev/null +++ b/src/black/mode.py @@ -0,0 +1,123 @@ +"""Data structures configuring Black behavior. + +Mostly around Python language feature support per version and Black configuration +chosen by the user. +""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, Set + +from black.const import DEFAULT_LINE_LENGTH + + +class TargetVersion(Enum): + PY27 = 2 + PY33 = 3 + PY34 = 4 + PY35 = 5 + PY36 = 6 + PY37 = 7 + PY38 = 8 + PY39 = 9 + + def is_python2(self) -> bool: + return self is TargetVersion.PY27 + + +class Feature(Enum): + # All string literals are unicode + UNICODE_LITERALS = 1 + F_STRINGS = 2 + NUMERIC_UNDERSCORES = 3 + TRAILING_COMMA_IN_CALL = 4 + TRAILING_COMMA_IN_DEF = 5 + # The following two feature-flags are mutually exclusive, and exactly one should be + # set for every version of python. + ASYNC_IDENTIFIERS = 6 + ASYNC_KEYWORDS = 7 + ASSIGNMENT_EXPRESSIONS = 8 + POS_ONLY_ARGUMENTS = 9 + RELAXED_DECORATORS = 10 + FORCE_OPTIONAL_PARENTHESES = 50 + + +VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { + TargetVersion.PY27: {Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY35: { + Feature.UNICODE_LITERALS, + Feature.TRAILING_COMMA_IN_CALL, + Feature.ASYNC_IDENTIFIERS, + }, + TargetVersion.PY36: { + Feature.UNICODE_LITERALS, + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_IDENTIFIERS, + }, + TargetVersion.PY37: { + Feature.UNICODE_LITERALS, + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + }, + TargetVersion.PY38: { + Feature.UNICODE_LITERALS, + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.POS_ONLY_ARGUMENTS, + }, + TargetVersion.PY39: { + Feature.UNICODE_LITERALS, + Feature.F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + }, +} + + +def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> bool: + return all(feature in VERSION_TO_FEATURES[version] for version in target_versions) + + +@dataclass +class Mode: + target_versions: Set[TargetVersion] = field(default_factory=set) + line_length: int = DEFAULT_LINE_LENGTH + string_normalization: bool = True + is_pyi: bool = False + magic_trailing_comma: bool = True + experimental_string_processing: bool = False + + def get_cache_key(self) -> str: + if self.target_versions: + version_str = ",".join( + str(version.value) + for version in sorted(self.target_versions, key=lambda v: v.value) + ) + else: + version_str = "-" + parts = [ + version_str, + str(self.line_length), + str(int(self.string_normalization)), + str(int(self.is_pyi)), + str(int(self.magic_trailing_comma)), + str(int(self.experimental_string_processing)), + ] + return ".".join(parts) diff --git a/src/black/nodes.py b/src/black/nodes.py new file mode 100644 index 00000000000..e0db9a42426 --- /dev/null +++ b/src/black/nodes.py @@ -0,0 +1,843 @@ +""" +blib2to3 Node/Leaf transformation-related utility functions. +""" + +import sys +from typing import ( + Collection, + Generic, + Iterator, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) + +if sys.version_info < (3, 8): + from typing_extensions import Final +else: + from typing import Final + +# lib2to3 fork +from blib2to3.pytree import Node, Leaf, type_repr +from blib2to3 import pygram +from blib2to3.pgen2 import token + +from black.cache import CACHE_DIR +from black.strings import has_triple_quotes + + +pygram.initialize(CACHE_DIR) +syms = pygram.python_symbols + + +# types +T = TypeVar("T") +LN = Union[Leaf, Node] +LeafID = int +NodeType = int + + +WHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE} +STATEMENT: Final = { + syms.if_stmt, + syms.while_stmt, + syms.for_stmt, + syms.try_stmt, + syms.except_clause, + syms.with_stmt, + syms.funcdef, + syms.classdef, +} +STANDALONE_COMMENT: Final = 153 +token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT" +LOGIC_OPERATORS: Final = {"and", "or"} +COMPARATORS: Final = { + token.LESS, + token.GREATER, + token.EQEQUAL, + token.NOTEQUAL, + token.LESSEQUAL, + token.GREATEREQUAL, +} +MATH_OPERATORS: Final = { + token.VBAR, + token.CIRCUMFLEX, + token.AMPER, + token.LEFTSHIFT, + token.RIGHTSHIFT, + token.PLUS, + token.MINUS, + token.STAR, + token.SLASH, + token.DOUBLESLASH, + token.PERCENT, + token.AT, + token.TILDE, + token.DOUBLESTAR, +} +STARS: Final = {token.STAR, token.DOUBLESTAR} +VARARGS_SPECIALS: Final = STARS | {token.SLASH} +VARARGS_PARENTS: Final = { + syms.arglist, + syms.argument, # double star in arglist + syms.trailer, # single argument to call + syms.typedargslist, + syms.varargslist, # lambdas +} +UNPACKING_PARENTS: Final = { + syms.atom, # single element of a list or set literal + syms.dictsetmaker, + syms.listmaker, + syms.testlist_gexp, + syms.testlist_star_expr, +} +TEST_DESCENDANTS: Final = { + syms.test, + syms.lambdef, + syms.or_test, + syms.and_test, + syms.not_test, + syms.comparison, + syms.star_expr, + syms.expr, + syms.xor_expr, + syms.and_expr, + syms.shift_expr, + syms.arith_expr, + syms.trailer, + syms.term, + syms.power, +} +ASSIGNMENTS: Final = { + "=", + "+=", + "-=", + "*=", + "@=", + "/=", + "%=", + "&=", + "|=", + "^=", + "<<=", + ">>=", + "**=", + "//=", +} + +IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist} +BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE} +OPENING_BRACKETS = set(BRACKET.keys()) +CLOSING_BRACKETS = set(BRACKET.values()) +BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS +ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT} + + +class Visitor(Generic[T]): + """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" + + def visit(self, node: LN) -> Iterator[T]: + """Main method to visit `node` and its children. + + It tries to find a `visit_*()` method for the given `node.type`, like + `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. + If no dedicated `visit_*()` method is found, chooses `visit_default()` + instead. + + Then yields objects of type `T` from the selected visitor. + """ + if node.type < 256: + name = token.tok_name[node.type] + else: + name = str(type_repr(node.type)) + # We explicitly branch on whether a visitor exists (instead of + # using self.visit_default as the default arg to getattr) in order + # to save needing to create a bound method object and so mypyc can + # generate a native call to visit_default. + visitf = getattr(self, f"visit_{name}", None) + if visitf: + yield from visitf(node) + else: + yield from self.visit_default(node) + + def visit_default(self, node: LN) -> Iterator[T]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" + if isinstance(node, Node): + for child in node.children: + yield from self.visit(child) + + +def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 + """Return whitespace prefix if needed for the given `leaf`. + + `complex_subscript` signals whether the given leaf is part of a subscription + which has non-trivial arguments, like arithmetic expressions or function calls. + """ + NO = "" + SPACE = " " + DOUBLESPACE = " " + t = leaf.type + p = leaf.parent + v = leaf.value + if t in ALWAYS_NO_SPACE: + return NO + + if t == token.COMMENT: + return DOUBLESPACE + + assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}" + if t == token.COLON and p.type not in { + syms.subscript, + syms.subscriptlist, + syms.sliceop, + }: + return NO + + prev = leaf.prev_sibling + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type in OPENING_BRACKETS: + return NO + + if t == token.COLON: + if prevp.type == token.COLON: + return NO + + elif prevp.type != token.COMMA and not complex_subscript: + return NO + + return SPACE + + if prevp.type == token.EQUAL: + if prevp.parent: + if prevp.parent.type in { + syms.arglist, + syms.argument, + syms.parameters, + syms.varargslist, + }: + return NO + + elif prevp.parent.type == syms.typedargslist: + # A bit hacky: if the equal sign has whitespace, it means we + # previously found it's a typed argument. So, we're using + # that, too. + return prevp.prefix + + elif prevp.type in VARARGS_SPECIALS: + if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS): + return NO + + elif prevp.type == token.COLON: + if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}: + return SPACE if complex_subscript else NO + + elif ( + prevp.parent + and prevp.parent.type == syms.factor + and prevp.type in MATH_OPERATORS + ): + return NO + + elif ( + prevp.type == token.RIGHTSHIFT + and prevp.parent + and prevp.parent.type == syms.shift_expr + and prevp.prev_sibling + and prevp.prev_sibling.type == token.NAME + and prevp.prev_sibling.value == "print" # type: ignore + ): + # Python 2 print chevron + return NO + elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator: + # no space in decorators + return NO + + elif prev.type in OPENING_BRACKETS: + return NO + + if p.type in {syms.parameters, syms.arglist}: + # untyped function signatures or calls + if not prev or prev.type != token.COMMA: + return NO + + elif p.type == syms.varargslist: + # lambdas + if prev and prev.type != token.COMMA: + return NO + + elif p.type == syms.typedargslist: + # typed function signatures + if not prev: + return NO + + if t == token.EQUAL: + if prev.type != syms.tname: + return NO + + elif prev.type == token.EQUAL: + # A bit hacky: if the equal sign has whitespace, it means we + # previously found it's a typed argument. So, we're using that, too. + return prev.prefix + + elif prev.type != token.COMMA: + return NO + + elif p.type == syms.tname: + # type names + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type != token.COMMA: + return NO + + elif p.type == syms.trailer: + # attributes and calls + if t == token.LPAR or t == token.RPAR: + return NO + + if not prev: + if t == token.DOT: + prevp = preceding_leaf(p) + if not prevp or prevp.type != token.NUMBER: + return NO + + elif t == token.LSQB: + return NO + + elif prev.type != token.COMMA: + return NO + + elif p.type == syms.argument: + # single argument + if t == token.EQUAL: + return NO + + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type == token.LPAR: + return NO + + elif prev.type in {token.EQUAL} | VARARGS_SPECIALS: + return NO + + elif p.type == syms.decorator: + # decorators + return NO + + elif p.type == syms.dotted_name: + if prev: + return NO + + prevp = preceding_leaf(p) + if not prevp or prevp.type == token.AT or prevp.type == token.DOT: + return NO + + elif p.type == syms.classdef: + if t == token.LPAR: + return NO + + if prev and prev.type == token.LPAR: + return NO + + elif p.type in {syms.subscript, syms.sliceop}: + # indexing + if not prev: + assert p.parent is not None, "subscripts are always parented" + if p.parent.type == syms.subscriptlist: + return SPACE + + return NO + + elif not complex_subscript: + return NO + + elif p.type == syms.atom: + if prev and t == token.DOT: + # dots, but not the first one. + return NO + + elif p.type == syms.dictsetmaker: + # dict unpacking + if prev and prev.type == token.DOUBLESTAR: + return NO + + elif p.type in {syms.factor, syms.star_expr}: + # unary ops + if not prev: + prevp = preceding_leaf(p) + if not prevp or prevp.type in OPENING_BRACKETS: + return NO + + prevp_parent = prevp.parent + assert prevp_parent is not None + if prevp.type == token.COLON and prevp_parent.type in { + syms.subscript, + syms.sliceop, + }: + return NO + + elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument: + return NO + + elif t in {token.NAME, token.NUMBER, token.STRING}: + return NO + + elif p.type == syms.import_from: + if t == token.DOT: + if prev and prev.type == token.DOT: + return NO + + elif t == token.NAME: + if v == "import": + return SPACE + + if prev and prev.type == token.DOT: + return NO + + elif p.type == syms.sliceop: + return NO + + return SPACE + + +def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: + """Return the first leaf that precedes `node`, if any.""" + while node: + res = node.prev_sibling + if res: + if isinstance(res, Leaf): + return res + + try: + return list(res.leaves())[-1] + + except IndexError: + return None + + node = node.parent + return None + + +def prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool: + """Return if the `node` and its previous siblings match types against the provided + list of tokens; the provided `node`has its type matched against the last element in + the list. `None` can be used as the first element to declare that the start of the + list is anchored at the start of its parent's children.""" + if not tokens: + return True + if tokens[-1] is None: + return node is None + if not node: + return False + if node.type != tokens[-1]: + return False + return prev_siblings_are(node.prev_sibling, tokens[:-1]) + + +def last_two_except(leaves: List[Leaf], omit: Collection[LeafID]) -> Tuple[Leaf, Leaf]: + """Return (penultimate, last) leaves skipping brackets in `omit` and contents.""" + stop_after = None + last = None + for leaf in reversed(leaves): + if stop_after: + if leaf is stop_after: + stop_after = None + continue + + if last: + return leaf, last + + if id(leaf) in omit: + stop_after = leaf.opening_bracket + else: + last = leaf + else: + raise LookupError("Last two leaves were also skipped") + + +def parent_type(node: Optional[LN]) -> Optional[NodeType]: + """ + Returns: + @node.parent.type, if @node is not None and has a parent. + OR + None, otherwise. + """ + if node is None or node.parent is None: + return None + + return node.parent.type + + +def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]: + """Return the child of `ancestor` that contains `descendant`.""" + node: Optional[LN] = descendant + while node and node.parent != ancestor: + node = node.parent + return node + + +def replace_child(old_child: LN, new_child: LN) -> None: + """ + Side Effects: + * If @old_child.parent is set, replace @old_child with @new_child in + @old_child's underlying Node structure. + OR + * Otherwise, this function does nothing. + """ + parent = old_child.parent + if not parent: + return + + child_idx = old_child.remove() + if child_idx is not None: + parent.insert_child(child_idx, new_child) + + +def container_of(leaf: Leaf) -> LN: + """Return `leaf` or one of its ancestors that is the topmost container of it. + + By "container" we mean a node where `leaf` is the very first child. + """ + same_prefix = leaf.prefix + container: LN = leaf + while container: + parent = container.parent + if parent is None: + break + + if parent.children[0].prefix != same_prefix: + break + + if parent.type == syms.file_input: + break + + if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS: + break + + container = parent + return container + + +def first_leaf_column(node: Node) -> Optional[int]: + """Returns the column of the first leaf child of a node.""" + for child in node.children: + if isinstance(child, Leaf): + return child.column + return None + + +def first_child_is_arith(node: Node) -> bool: + """Whether first child is an arithmetic or a binary arithmetic expression""" + expr_types = { + syms.arith_expr, + syms.shift_expr, + syms.xor_expr, + syms.and_expr, + } + return bool(node.children and node.children[0].type in expr_types) + + +def is_docstring(leaf: Leaf) -> bool: + if prev_siblings_are( + leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] + ): + return True + + # Multiline docstring on the same line as the `def`. + if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]): + # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python + # grammar. We're safe to return True without further checks. + return True + + return False + + +def is_empty_tuple(node: LN) -> bool: + """Return True if `node` holds an empty tuple.""" + return ( + node.type == syms.atom + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) + + +def is_one_tuple(node: LN) -> bool: + """Return True if `node` holds a tuple with one element, with or without parens.""" + if node.type == syms.atom: + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA + + return ( + node.type in IMPLICIT_TUPLE + and len(node.children) == 2 + and node.children[1].type == token.COMMA + ) + + +def is_one_tuple_between(opening: Leaf, closing: Leaf, leaves: List[Leaf]) -> bool: + """Return True if content between `opening` and `closing` looks like a one-tuple.""" + if opening.type != token.LPAR and closing.type != token.RPAR: + return False + + depth = closing.bracket_depth + 1 + for _opening_index, leaf in enumerate(leaves): + if leaf is opening: + break + + else: + raise LookupError("Opening paren not found in `leaves`") + + commas = 0 + _opening_index += 1 + for leaf in leaves[_opening_index:]: + if leaf is closing: + break + + bracket_depth = leaf.bracket_depth + if bracket_depth == depth and leaf.type == token.COMMA: + commas += 1 + if leaf.parent and leaf.parent.type in { + syms.arglist, + syms.typedargslist, + }: + commas += 1 + break + + return commas < 2 + + +def is_walrus_assignment(node: LN) -> bool: + """Return True iff `node` is of the shape ( test := test )""" + inner = unwrap_singleton_parenthesis(node) + return inner is not None and inner.type == syms.namedexpr_test + + +def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool: + """Return True iff `node` is a trailer valid in a simple decorator""" + return node.type == syms.trailer and ( + ( + len(node.children) == 2 + and node.children[0].type == token.DOT + and node.children[1].type == token.NAME + ) + # last trailer can be an argument-less parentheses pair + or ( + last + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) + # last trailer can be arguments + or ( + last + and len(node.children) == 3 + and node.children[0].type == token.LPAR + # and node.children[1].type == syms.argument + and node.children[2].type == token.RPAR + ) + ) + + +def is_simple_decorator_expression(node: LN) -> bool: + """Return True iff `node` could be a 'dotted name' decorator + + This function takes the node of the 'namedexpr_test' of the new decorator + grammar and test if it would be valid under the old decorator grammar. + + The old grammar was: decorator: @ dotted_name [arguments] NEWLINE + The new grammar is : decorator: @ namedexpr_test NEWLINE + """ + if node.type == token.NAME: + return True + if node.type == syms.power: + if node.children: + return ( + node.children[0].type == token.NAME + and all(map(is_simple_decorator_trailer, node.children[1:-1])) + and ( + len(node.children) < 2 + or is_simple_decorator_trailer(node.children[-1], last=True) + ) + ) + return False + + +def is_yield(node: LN) -> bool: + """Return True if `node` holds a `yield` or `yield from` expression.""" + if node.type == syms.yield_expr: + return True + + if node.type == token.NAME and node.value == "yield": # type: ignore + return True + + if node.type != syms.atom: + return False + + if len(node.children) != 3: + return False + + lpar, expr, rpar = node.children + if lpar.type == token.LPAR and rpar.type == token.RPAR: + return is_yield(expr) + + return False + + +def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool: + """Return True if `leaf` is a star or double star in a vararg or kwarg. + + If `within` includes VARARGS_PARENTS, this applies to function signatures. + If `within` includes UNPACKING_PARENTS, it applies to right hand-side + extended iterable unpacking (PEP 3132) and additional unpacking + generalizations (PEP 448). + """ + if leaf.type not in VARARGS_SPECIALS or not leaf.parent: + return False + + p = leaf.parent + if p.type == syms.star_expr: + # Star expressions are also used as assignment targets in extended + # iterable unpacking (PEP 3132). See what its parent is instead. + if not p.parent: + return False + + p = p.parent + + return p.type in within + + +def is_multiline_string(leaf: Leaf) -> bool: + """Return True if `leaf` is a multiline string that actually spans many lines.""" + return has_triple_quotes(leaf.value) and "\n" in leaf.value + + +def is_stub_suite(node: Node) -> bool: + """Return True if `node` is a suite with a stub body.""" + if ( + len(node.children) != 4 + or node.children[0].type != token.NEWLINE + or node.children[1].type != token.INDENT + or node.children[3].type != token.DEDENT + ): + return False + + return is_stub_body(node.children[2]) + + +def is_stub_body(node: LN) -> bool: + """Return True if `node` is a simple statement containing an ellipsis.""" + if not isinstance(node, Node) or node.type != syms.simple_stmt: + return False + + if len(node.children) != 2: + return False + + child = node.children[0] + return ( + child.type == syms.atom + and len(child.children) == 3 + and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) + ) + + +def is_atom_with_invisible_parens(node: LN) -> bool: + """Given a `LN`, determines whether it's an atom `node` with invisible + parens. Useful in dedupe-ing and normalizing parens. + """ + if isinstance(node, Leaf) or node.type != syms.atom: + return False + + first, last = node.children[0], node.children[-1] + return ( + isinstance(first, Leaf) + and first.type == token.LPAR + and first.value == "" + and isinstance(last, Leaf) + and last.type == token.RPAR + and last.value == "" + ) + + +def is_empty_par(leaf: Leaf) -> bool: + return is_empty_lpar(leaf) or is_empty_rpar(leaf) + + +def is_empty_lpar(leaf: Leaf) -> bool: + return leaf.type == token.LPAR and leaf.value == "" + + +def is_empty_rpar(leaf: Leaf) -> bool: + return leaf.type == token.RPAR and leaf.value == "" + + +def is_import(leaf: Leaf) -> bool: + """Return True if the given leaf starts an import statement.""" + p = leaf.parent + t = leaf.type + v = leaf.value + return bool( + t == token.NAME + and ( + (v == "import" and p and p.type == syms.import_name) + or (v == "from" and p and p.type == syms.import_from) + ) + ) + + +def is_type_comment(leaf: Leaf, suffix: str = "") -> bool: + """Return True if the given leaf is a special comment. + Only returns true for type comments for now.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix) + + +def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: + """Wrap `child` in parentheses. + + This replaces `child` with an atom holding the parentheses and the old + child. That requires moving the prefix. + + If `visible` is False, the leaves will be valueless (and thus invisible). + """ + lpar = Leaf(token.LPAR, "(" if visible else "") + rpar = Leaf(token.RPAR, ")" if visible else "") + prefix = child.prefix + child.prefix = "" + index = child.remove() or 0 + new_child = Node(syms.atom, [lpar, child, rpar]) + new_child.prefix = prefix + parent.insert_child(index, new_child) + + +def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]: + """Returns `wrapped` if `node` is of the shape ( wrapped ). + + Parenthesis can be optional. Returns None otherwise""" + if len(node.children) != 3: + return None + + lpar, wrapped, rpar = node.children + if not (lpar.type == token.LPAR and rpar.type == token.RPAR): + return None + + return wrapped + + +def ensure_visible(leaf: Leaf) -> None: + """Make sure parentheses are visible. + + They could be invisible as part of some statements (see + :func:`normalize_invisible_parens` and :func:`visit_import_from`). + """ + if leaf.type == token.LPAR: + leaf.value = "(" + elif leaf.type == token.RPAR: + leaf.value = ")" diff --git a/src/black/numerics.py b/src/black/numerics.py new file mode 100644 index 00000000000..cb1c83e7b78 --- /dev/null +++ b/src/black/numerics.py @@ -0,0 +1,65 @@ +""" +Formatting numeric literals. +""" +from blib2to3.pytree import Leaf + + +def format_hex(text: str) -> str: + """ + Formats a hexadecimal string like "0x12B3" + """ + before, after = text[:2], text[2:] + return f"{before}{after.upper()}" + + +def format_scientific_notation(text: str) -> str: + """Formats a numeric string utilizing scentific notation""" + before, after = text.split("e") + sign = "" + if after.startswith("-"): + after = after[1:] + sign = "-" + elif after.startswith("+"): + after = after[1:] + before = format_float_or_int_string(before) + return f"{before}e{sign}{after}" + + +def format_long_or_complex_number(text: str) -> str: + """Formats a long or complex string like `10L` or `10j`""" + number = text[:-1] + suffix = text[-1] + # Capitalize in "2L" because "l" looks too similar to "1". + if suffix == "l": + suffix = "L" + return f"{format_float_or_int_string(number)}{suffix}" + + +def format_float_or_int_string(text: str) -> str: + """Formats a float string like "1.0".""" + if "." not in text: + return text + + before, after = text.split(".") + return f"{before or 0}.{after or 0}" + + +def normalize_numeric_literal(leaf: Leaf) -> None: + """Normalizes numeric (float, int, and complex) literals. + + All letters used in the representation are normalized to lowercase (except + in Python 2 long literals). + """ + text = leaf.value.lower() + if text.startswith(("0o", "0b")): + # Leave octal and binary literals alone. + pass + elif text.startswith("0x"): + text = format_hex(text) + elif "e" in text: + text = format_scientific_notation(text) + elif text.endswith(("j", "l")): + text = format_long_or_complex_number(text) + else: + text = format_float_or_int_string(text) + leaf.value = text diff --git a/src/black/output.py b/src/black/output.py new file mode 100644 index 00000000000..6831524090d --- /dev/null +++ b/src/black/output.py @@ -0,0 +1,83 @@ +"""Nice output for Black. + +The double calls are for patching purposes in tests. +""" + +from typing import Any, Optional +from mypy_extensions import mypyc_attr +import tempfile + +from click import echo, style + + +def _out(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + if message is not None: + if "bold" not in styles: + styles["bold"] = True + message = style(message, **styles) + echo(message, nl=nl, err=True) + + +def _err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + if message is not None: + if "fg" not in styles: + styles["fg"] = "red" + message = style(message, **styles) + echo(message, nl=nl, err=True) + + +def out(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + _out(message, nl=nl, **styles) + + +def err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: + _err(message, nl=nl, **styles) + + +def diff(a: str, b: str, a_name: str, b_name: str) -> str: + """Return a unified diff string between strings `a` and `b`.""" + import difflib + + a_lines = [line for line in a.splitlines(keepends=True)] + b_lines = [line for line in b.splitlines(keepends=True)] + diff_lines = [] + for line in difflib.unified_diff( + a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 + ): + # Work around https://bugs.python.org/issue2142 + # See https://www.gnu.org/software/diffutils/manual/html_node/Incomplete-Lines.html + if line[-1] == "\n": + diff_lines.append(line) + else: + diff_lines.append(line + "\n") + diff_lines.append("\\ No newline at end of file\n") + return "".join(diff_lines) + + +def color_diff(contents: str) -> str: + """Inject the ANSI color codes to the diff.""" + lines = contents.split("\n") + for i, line in enumerate(lines): + if line.startswith("+++") or line.startswith("---"): + line = "\033[1;37m" + line + "\033[0m" # bold white, reset + elif line.startswith("@@"): + line = "\033[36m" + line + "\033[0m" # cyan, reset + elif line.startswith("+"): + line = "\033[32m" + line + "\033[0m" # green, reset + elif line.startswith("-"): + line = "\033[31m" + line + "\033[0m" # red, reset + lines[i] = line + return "\n".join(lines) + + +@mypyc_attr(patchable=True) +def dump_to_file(*output: str, ensure_final_newline: bool = True) -> str: + """Dump `output` to a temporary file. Return path to the file.""" + with tempfile.NamedTemporaryFile( + mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8" + ) as f: + for lines in output: + f.write(lines) + if ensure_final_newline and lines and lines[-1] != "\n": + f.write("\n") + return f.name diff --git a/src/black/parsing.py b/src/black/parsing.py new file mode 100644 index 00000000000..8e9feea9120 --- /dev/null +++ b/src/black/parsing.py @@ -0,0 +1,215 @@ +""" +Parse Python code and perform AST validation. +""" +import ast +import sys +from typing import Iterable, Iterator, List, Set, Union + +# lib2to3 fork +from blib2to3.pytree import Node, Leaf +from blib2to3 import pygram, pytree +from blib2to3.pgen2 import driver +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.parse import ParseError + +from black.mode import TargetVersion, Feature, supports_feature +from black.nodes import syms + +try: + from typed_ast import ast3, ast27 +except ImportError: + if sys.version_info < (3, 8): + print( + "The typed_ast package is required but not installed.\n" + "You can upgrade to Python 3.8+ or install typed_ast with\n" + "`python3 -m pip install typed-ast`.", + file=sys.stderr, + ) + sys.exit(1) + else: + ast3 = ast27 = ast + + +class InvalidInput(ValueError): + """Raised when input source code fails all parse attempts.""" + + +def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: + if not target_versions: + # No target_version specified, so try all grammars. + return [ + # Python 3.7+ + pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, + # Python 3.0-3.6 + pygram.python_grammar_no_print_statement_no_exec_statement, + # Python 2.7 with future print_function import + pygram.python_grammar_no_print_statement, + # Python 2.7 + pygram.python_grammar, + ] + + if all(version.is_python2() for version in target_versions): + # Python 2-only code, so try Python 2 grammars. + return [ + # Python 2.7 with future print_function import + pygram.python_grammar_no_print_statement, + # Python 2.7 + pygram.python_grammar, + ] + + # Python 3-compatible code, so only try Python 3 grammar. + grammars = [] + # If we have to parse both, try to parse async as a keyword first + if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS): + # Python 3.7+ + grammars.append( + pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords + ) + if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): + # Python 3.0-3.6 + grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) + # At least one of the above branches must have been taken, because every Python + # version has exactly one of the two 'ASYNC_*' flags + return grammars + + +def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node: + """Given a string with source, return the lib2to3 Node.""" + if not src_txt.endswith("\n"): + src_txt += "\n" + + for grammar in get_grammars(set(target_versions)): + drv = driver.Driver(grammar, pytree.convert) + try: + result = drv.parse_string(src_txt, True) + break + + except ParseError as pe: + lineno, column = pe.context[1] + lines = src_txt.splitlines() + try: + faulty_line = lines[lineno - 1] + except IndexError: + faulty_line = "" + exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}") + else: + raise exc from None + + if isinstance(result, Leaf): + result = Node(syms.file_input, [result]) + return result + + +def lib2to3_unparse(node: Node) -> str: + """Given a lib2to3 node, return its string representation.""" + code = str(node) + return code + + +def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]: + filename = "" + if sys.version_info >= (3, 8): + # TODO: support Python 4+ ;) + for minor_version in range(sys.version_info[1], 4, -1): + try: + return ast.parse(src, filename, feature_version=(3, minor_version)) + except SyntaxError: + continue + else: + for feature_version in (7, 6): + try: + return ast3.parse(src, filename, feature_version=feature_version) + except SyntaxError: + continue + if ast27.__name__ == "ast": + raise SyntaxError( + "The requested source code has invalid Python 3 syntax.\n" + "If you are trying to format Python 2 files please reinstall Black" + " with the 'python2' extra: `python3 -m pip install black[python2]`." + ) + return ast27.parse(src) + + +def stringify_ast( + node: Union[ast.AST, ast3.AST, ast27.AST], depth: int = 0 +) -> Iterator[str]: + """Simple visitor generating strings to compare ASTs by content.""" + + node = fixup_ast_constants(node) + + yield f"{' ' * depth}{node.__class__.__name__}(" + + for field in sorted(node._fields): # noqa: F402 + # TypeIgnore has only one field 'lineno' which breaks this comparison + type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore) + if sys.version_info >= (3, 8): + type_ignore_classes += (ast.TypeIgnore,) + if isinstance(node, type_ignore_classes): + break + + try: + value = getattr(node, field) + except AttributeError: + continue + + yield f"{' ' * (depth+1)}{field}=" + + if isinstance(value, list): + for item in value: + # Ignore nested tuples within del statements, because we may insert + # parentheses and they change the AST. + if ( + field == "targets" + and isinstance(node, (ast.Delete, ast3.Delete, ast27.Delete)) + and isinstance(item, (ast.Tuple, ast3.Tuple, ast27.Tuple)) + ): + for item in item.elts: + yield from stringify_ast(item, depth + 2) + + elif isinstance(item, (ast.AST, ast3.AST, ast27.AST)): + yield from stringify_ast(item, depth + 2) + + elif isinstance(value, (ast.AST, ast3.AST, ast27.AST)): + yield from stringify_ast(value, depth + 2) + + else: + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. + # Note that when formatting Python 2 code, at least with Windows + # line-endings, docstrings can end up here as bytes instead of + # str so make sure that we handle both cases. + if ( + isinstance(node, ast.Constant) + and field == "value" + and isinstance(value, (str, bytes)) + ): + lineend = "\n" if isinstance(value, str) else b"\n" + # To normalize, we strip any leading and trailing space from + # each line... + stripped = [line.strip() for line in value.splitlines()] + normalized = lineend.join(stripped) # type: ignore[attr-defined] + # ...and remove any blank lines at the beginning and end of + # the whole string + normalized = normalized.strip() + else: + normalized = value + yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" + + yield f"{' ' * depth}) # /{node.__class__.__name__}" + + +def fixup_ast_constants( + node: Union[ast.AST, ast3.AST, ast27.AST] +) -> Union[ast.AST, ast3.AST, ast27.AST]: + """Map ast nodes deprecated in 3.8 to Constant.""" + if isinstance(node, (ast.Str, ast3.Str, ast27.Str, ast.Bytes, ast3.Bytes)): + return ast.Constant(value=node.s) + + if isinstance(node, (ast.Num, ast3.Num, ast27.Num)): + return ast.Constant(value=node.n) + + if isinstance(node, (ast.NameConstant, ast3.NameConstant)): + return ast.Constant(value=node.value) + + return node diff --git a/src/black/report.py b/src/black/report.py new file mode 100644 index 00000000000..8fc5da2e167 --- /dev/null +++ b/src/black/report.py @@ -0,0 +1,100 @@ +""" +Summarize Black runs to users. +""" +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from click import style + +from black.output import out, err + + +class Changed(Enum): + NO = 0 + CACHED = 1 + YES = 2 + + +@dataclass +class Report: + """Provides a reformatting counter. Can be rendered with `str(report)`.""" + + check: bool = False + diff: bool = False + quiet: bool = False + verbose: bool = False + change_count: int = 0 + same_count: int = 0 + failure_count: int = 0 + + def done(self, src: Path, changed: Changed) -> None: + """Increment the counter for successful reformatting. Write out a message.""" + if changed is Changed.YES: + reformatted = "would reformat" if self.check or self.diff else "reformatted" + if self.verbose or not self.quiet: + out(f"{reformatted} {src}") + self.change_count += 1 + else: + if self.verbose: + if changed is Changed.NO: + msg = f"{src} already well formatted, good job." + else: + msg = f"{src} wasn't modified on disk since last run." + out(msg, bold=False) + self.same_count += 1 + + def failed(self, src: Path, message: str) -> None: + """Increment the counter for failed reformatting. Write out a message.""" + err(f"error: cannot format {src}: {message}") + self.failure_count += 1 + + def path_ignored(self, path: Path, message: str) -> None: + if self.verbose: + out(f"{path} ignored: {message}", bold=False) + + @property + def return_code(self) -> int: + """Return the exit code that the app should use. + + This considers the current state of changed files and failures: + - if there were any failures, return 123; + - if any files were changed and --check is being used, return 1; + - otherwise return 0. + """ + # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with + # 126 we have special return codes reserved by the shell. + if self.failure_count: + return 123 + + elif self.change_count and self.check: + return 1 + + return 0 + + def __str__(self) -> str: + """Render a color report of the current state. + + Use `click.unstyle` to remove colors. + """ + if self.check or self.diff: + reformatted = "would be reformatted" + unchanged = "would be left unchanged" + failed = "would fail to reformat" + else: + reformatted = "reformatted" + unchanged = "left unchanged" + failed = "failed to reformat" + report = [] + if self.change_count: + s = "s" if self.change_count > 1 else "" + report.append( + style(f"{self.change_count} file{s} {reformatted}", bold=True) + ) + if self.same_count: + s = "s" if self.same_count > 1 else "" + report.append(f"{self.same_count} file{s} {unchanged}") + if self.failure_count: + s = "s" if self.failure_count > 1 else "" + report.append(style(f"{self.failure_count} file{s} {failed}", fg="red")) + return ", ".join(report) + "." diff --git a/src/black/rusty.py b/src/black/rusty.py new file mode 100644 index 00000000000..822e3d7858a --- /dev/null +++ b/src/black/rusty.py @@ -0,0 +1,28 @@ +"""An error-handling model influenced by that used by the Rust programming language + +See https://doc.rust-lang.org/book/ch09-00-error-handling.html. +""" +from typing import Generic, TypeVar, Union + + +T = TypeVar("T") +E = TypeVar("E", bound=Exception) + + +class Ok(Generic[T]): + def __init__(self, value: T) -> None: + self._value = value + + def ok(self) -> T: + return self._value + + +class Err(Generic[E]): + def __init__(self, e: E) -> None: + self._e = e + + def err(self) -> E: + return self._e + + +Result = Union[Ok[T], Err[E]] diff --git a/src/black/strings.py b/src/black/strings.py new file mode 100644 index 00000000000..5b443ddebc9 --- /dev/null +++ b/src/black/strings.py @@ -0,0 +1,216 @@ +""" +Simple formatting on strings. Further string formatting code is in trans.py. +""" + +import regex as re +import sys +from typing import List, Pattern + + +STRING_PREFIX_CHARS = "furbFURB" # All possible string prefix characters. + + +def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str: + """Replace `regex` with `replacement` twice on `original`. + + This is used by string normalization to perform replaces on + overlapping matches. + """ + return regex.sub(replacement, regex.sub(replacement, original)) + + +def has_triple_quotes(string: str) -> bool: + """ + Returns: + True iff @string starts with three quotation characters. + """ + raw_string = string.lstrip(STRING_PREFIX_CHARS) + return raw_string[:3] in {'"""', "'''"} + + +def lines_with_leading_tabs_expanded(s: str) -> List[str]: + """ + Splits string into lines and expands only leading tabs (following the normal + Python rules) + """ + lines = [] + for line in s.splitlines(): + # Find the index of the first non-whitespace character after a string of + # whitespace that includes at least one tab + match = re.match(r"\s*\t+\s*(\S)", line) + if match: + first_non_whitespace_idx = match.start(1) + + lines.append( + line[:first_non_whitespace_idx].expandtabs() + + line[first_non_whitespace_idx:] + ) + else: + lines.append(line) + return lines + + +def fix_docstring(docstring: str, prefix: str) -> str: + # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation + if not docstring: + return "" + lines = lines_with_leading_tabs_expanded(docstring) + # Determine minimum indentation (first line doesn't count): + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + last_line_idx = len(lines) - 2 + for i, line in enumerate(lines[1:]): + stripped_line = line[indent:].rstrip() + if stripped_line or i == last_line_idx: + trimmed.append(prefix + stripped_line) + else: + trimmed.append("") + return "\n".join(trimmed) + + +def get_string_prefix(string: str) -> str: + """ + Pre-conditions: + * assert_is_leaf_string(@string) + + Returns: + @string's prefix (e.g. '', 'r', 'f', or 'rf'). + """ + assert_is_leaf_string(string) + + prefix = "" + prefix_idx = 0 + while string[prefix_idx] in STRING_PREFIX_CHARS: + prefix += string[prefix_idx].lower() + prefix_idx += 1 + + return prefix + + +def assert_is_leaf_string(string: str) -> None: + """ + Checks the pre-condition that @string has the format that you would expect + of `leaf.value` where `leaf` is some Leaf such that `leaf.type == + token.STRING`. A more precise description of the pre-conditions that are + checked are listed below. + + Pre-conditions: + * @string starts with either ', ", ', or " where + `set()` is some subset of `set(STRING_PREFIX_CHARS)`. + * @string ends with a quote character (' or "). + + Raises: + AssertionError(...) if the pre-conditions listed above are not + satisfied. + """ + dquote_idx = string.find('"') + squote_idx = string.find("'") + if -1 in [dquote_idx, squote_idx]: + quote_idx = max(dquote_idx, squote_idx) + else: + quote_idx = min(squote_idx, dquote_idx) + + assert ( + 0 <= quote_idx < len(string) - 1 + ), f"{string!r} is missing a starting quote character (' or \")." + assert string[-1] in ( + "'", + '"', + ), f"{string!r} is missing an ending quote character (' or \")." + assert set(string[:quote_idx]).issubset( + set(STRING_PREFIX_CHARS) + ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}." + + +def normalize_string_prefix(s: str, remove_u_prefix: bool = False) -> str: + """Make all string prefixes lowercase. + + If remove_u_prefix is given, also removes any u prefix from the string. + """ + match = re.match(r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", s, re.DOTALL) + assert match is not None, f"failed to match string {s!r}" + orig_prefix = match.group(1) + new_prefix = orig_prefix.replace("F", "f").replace("B", "b").replace("U", "u") + if remove_u_prefix: + new_prefix = new_prefix.replace("u", "") + return f"{new_prefix}{match.group(2)}" + + +def normalize_string_quotes(s: str) -> str: + """Prefer double quotes but only if it doesn't cause more escaping. + + Adds or removes backslashes as appropriate. Doesn't parse and fix + strings nested in f-strings. + """ + value = s.lstrip(STRING_PREFIX_CHARS) + if value[:3] == '"""': + return s + + elif value[:3] == "'''": + orig_quote = "'''" + new_quote = '"""' + elif value[0] == '"': + orig_quote = '"' + new_quote = "'" + else: + orig_quote = "'" + new_quote = '"' + first_quote_pos = s.find(orig_quote) + if first_quote_pos == -1: + return s # There's an internal error + + prefix = s[:first_quote_pos] + unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}") + escaped_new_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}") + escaped_orig_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}") + body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)] + if "r" in prefix.casefold(): + if unescaped_new_quote.search(body): + # There's at least one unescaped new_quote in this raw string + # so converting is impossible + return s + + # Do not introduce or remove backslashes in raw strings + new_body = body + else: + # remove unnecessary escapes + new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body) + if body != new_body: + # Consider the string without unnecessary escapes as the original + body = new_body + s = f"{prefix}{orig_quote}{body}{orig_quote}" + new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body) + new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body) + if "f" in prefix.casefold(): + matches = re.findall( + r""" + (?:[^{]|^)\{ # start of the string or a non-{ followed by a single { + ([^{].*?) # contents of the brackets except if begins with {{ + \}(?:[^}]|$) # A } followed by end of the string or a non-} + """, + new_body, + re.VERBOSE, + ) + for m in matches: + if "\\" in str(m): + # Do not introduce backslashes in interpolated expressions + return s + + if new_quote == '"""' and new_body[-1:] == '"': + # edge case: + new_body = new_body[:-1] + '\\"' + orig_escape_count = body.count("\\") + new_escape_count = new_body.count("\\") + if new_escape_count > orig_escape_count: + return s # Do not introduce more escaping + + if new_escape_count == orig_escape_count and orig_quote == '"': + return s # Prefer double quotes + + return f"{prefix}{new_quote}{new_body}{new_quote}" diff --git a/src/black/trans.py b/src/black/trans.py new file mode 100644 index 00000000000..055f33c0ee2 --- /dev/null +++ b/src/black/trans.py @@ -0,0 +1,1925 @@ +""" +String transformers that can split and merge strings. +""" +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass +import regex as re +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, +) + +from black.rusty import Result, Ok, Err + +from black.mode import Feature +from black.nodes import syms, replace_child, parent_type +from black.nodes import is_empty_par, is_empty_lpar, is_empty_rpar +from black.nodes import CLOSING_BRACKETS, STANDALONE_COMMENT +from black.lines import Line, append_leaves +from black.brackets import BracketMatchError +from black.comments import contains_pragma_comment +from black.strings import has_triple_quotes, get_string_prefix, assert_is_leaf_string +from black.strings import normalize_string_quotes + +from blib2to3.pytree import Leaf, Node +from blib2to3.pgen2 import token + + +class CannotTransform(Exception): + """Base class for errors raised by Transformers.""" + + +# types +T = TypeVar("T") +LN = Union[Leaf, Node] +Transformer = Callable[[Line, Collection[Feature]], Iterator[Line]] +Index = int +NodeType = int +ParserState = int +StringID = int +TResult = Result[T, CannotTransform] # (T)ransform Result +TMatchResult = TResult[Index] + + +def TErr(err_msg: str) -> Err[CannotTransform]: + """(T)ransform Err + + Convenience function used when working with the TResult type. + """ + cant_transform = CannotTransform(err_msg) + return Err(cant_transform) + + +@dataclass # type: ignore +class StringTransformer(ABC): + """ + An implementation of the Transformer protocol that relies on its + subclasses overriding the template methods `do_match(...)` and + `do_transform(...)`. + + This Transformer works exclusively on strings (for example, by merging + or splitting them). + + The following sections can be found among the docstrings of each concrete + StringTransformer subclass. + + Requirements: + Which requirements must be met of the given Line for this + StringTransformer to be applied? + + Transformations: + If the given Line meets all of the above requirements, which string + transformations can you expect to be applied to it by this + StringTransformer? + + Collaborations: + What contractual agreements does this StringTransformer have with other + StringTransfomers? Such collaborations should be eliminated/minimized + as much as possible. + """ + + line_length: int + normalize_strings: bool + __name__ = "StringTransformer" + + @abstractmethod + def do_match(self, line: Line) -> TMatchResult: + """ + Returns: + * Ok(string_idx) such that `line.leaves[string_idx]` is our target + string, if a match was able to be made. + OR + * Err(CannotTransform), if a match was not able to be made. + """ + + @abstractmethod + def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + """ + Yields: + * Ok(new_line) where new_line is the new transformed line. + OR + * Err(CannotTransform) if the transformation failed for some reason. The + `do_match(...)` template method should usually be used to reject + the form of the given Line, but in some cases it is difficult to + know whether or not a Line meets the StringTransformer's + requirements until the transformation is already midway. + + Side Effects: + This method should NOT mutate @line directly, but it MAY mutate the + Line's underlying Node structure. (WARNING: If the underlying Node + structure IS altered, then this method should NOT be allowed to + yield an CannotTransform after that point.) + """ + + def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]: + """ + StringTransformer instances have a call signature that mirrors that of + the Transformer type. + + Raises: + CannotTransform(...) if the concrete StringTransformer class is unable + to transform @line. + """ + # Optimization to avoid calling `self.do_match(...)` when the line does + # not contain any string. + if not any(leaf.type == token.STRING for leaf in line.leaves): + raise CannotTransform("There are no strings in this line.") + + match_result = self.do_match(line) + + if isinstance(match_result, Err): + cant_transform = match_result.err() + raise CannotTransform( + f"The string transformer {self.__class__.__name__} does not recognize" + " this line as one that it can transform." + ) from cant_transform + + string_idx = match_result.ok() + + for line_result in self.do_transform(line, string_idx): + if isinstance(line_result, Err): + cant_transform = line_result.err() + raise CannotTransform( + "StringTransformer failed while attempting to transform string." + ) from cant_transform + line = line_result.ok() + yield line + + +@dataclass +class CustomSplit: + """A custom (i.e. manual) string split. + + A single CustomSplit instance represents a single substring. + + Examples: + Consider the following string: + ``` + "Hi there friend." + " This is a custom" + f" string {split}." + ``` + + This string will correspond to the following three CustomSplit instances: + ``` + CustomSplit(False, 16) + CustomSplit(False, 17) + CustomSplit(True, 16) + ``` + """ + + has_prefix: bool + break_idx: int + + +class CustomSplitMapMixin: + """ + This mixin class is used to map merged strings to a sequence of + CustomSplits, which will then be used to re-split the strings iff none of + the resultant substrings go over the configured max line length. + """ + + _Key = Tuple[StringID, str] + _CUSTOM_SPLIT_MAP: Dict[_Key, Tuple[CustomSplit, ...]] = defaultdict(tuple) + + @staticmethod + def _get_key(string: str) -> "CustomSplitMapMixin._Key": + """ + Returns: + A unique identifier that is used internally to map @string to a + group of custom splits. + """ + return (id(string), string) + + def add_custom_splits( + self, string: str, custom_splits: Iterable[CustomSplit] + ) -> None: + """Custom Split Map Setter Method + + Side Effects: + Adds a mapping from @string to the custom splits @custom_splits. + """ + key = self._get_key(string) + self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits) + + def pop_custom_splits(self, string: str) -> List[CustomSplit]: + """Custom Split Map Getter Method + + Returns: + * A list of the custom splits that are mapped to @string, if any + exist. + OR + * [], otherwise. + + Side Effects: + Deletes the mapping between @string and its associated custom + splits (which are returned to the caller). + """ + key = self._get_key(string) + + custom_splits = self._CUSTOM_SPLIT_MAP[key] + del self._CUSTOM_SPLIT_MAP[key] + + return list(custom_splits) + + def has_custom_splits(self, string: str) -> bool: + """ + Returns: + True iff @string is associated with a set of custom splits. + """ + key = self._get_key(string) + return key in self._CUSTOM_SPLIT_MAP + + +class StringMerger(CustomSplitMapMixin, StringTransformer): + """StringTransformer that merges strings together. + + Requirements: + (A) The line contains adjacent strings such that ALL of the validation checks + listed in StringMerger.__validate_msg(...)'s docstring pass. + OR + (B) The line contains a string which uses line continuation backslashes. + + Transformations: + Depending on which of the two requirements above where met, either: + + (A) The string group associated with the target string is merged. + OR + (B) All line-continuation backslashes are removed from the target string. + + Collaborations: + StringMerger provides custom split information to StringSplitter. + """ + + def do_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + for (i, leaf) in enumerate(LL): + if ( + leaf.type == token.STRING + and is_valid_index(i + 1) + and LL[i + 1].type == token.STRING + ): + return Ok(i) + + if leaf.type == token.STRING and "\\\n" in leaf.value: + return Ok(i) + + return TErr("This line has no strings that need merging.") + + def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + new_line = line + rblc_result = self._remove_backslash_line_continuation_chars( + new_line, string_idx + ) + if isinstance(rblc_result, Ok): + new_line = rblc_result.ok() + + msg_result = self._merge_string_group(new_line, string_idx) + if isinstance(msg_result, Ok): + new_line = msg_result.ok() + + if isinstance(rblc_result, Err) and isinstance(msg_result, Err): + msg_cant_transform = msg_result.err() + rblc_cant_transform = rblc_result.err() + cant_transform = CannotTransform( + "StringMerger failed to merge any strings in this line." + ) + + # Chain the errors together using `__cause__`. + msg_cant_transform.__cause__ = rblc_cant_transform + cant_transform.__cause__ = msg_cant_transform + + yield Err(cant_transform) + else: + yield Ok(new_line) + + @staticmethod + def _remove_backslash_line_continuation_chars( + line: Line, string_idx: int + ) -> TResult[Line]: + """ + Merge strings that were split across multiple lines using + line-continuation backslashes. + + Returns: + Ok(new_line), if @line contains backslash line-continuation + characters. + OR + Err(CannotTransform), otherwise. + """ + LL = line.leaves + + string_leaf = LL[string_idx] + if not ( + string_leaf.type == token.STRING + and "\\\n" in string_leaf.value + and not has_triple_quotes(string_leaf.value) + ): + return TErr( + f"String leaf {string_leaf} does not contain any backslash line" + " continuation characters." + ) + + new_line = line.clone() + new_line.comments = line.comments.copy() + append_leaves(new_line, line, LL) + + new_string_leaf = new_line.leaves[string_idx] + new_string_leaf.value = new_string_leaf.value.replace("\\\n", "") + + return Ok(new_line) + + def _merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]: + """ + Merges string group (i.e. set of adjacent strings) where the first + string in the group is `line.leaves[string_idx]`. + + Returns: + Ok(new_line), if ALL of the validation checks found in + __validate_msg(...) pass. + OR + Err(CannotTransform), otherwise. + """ + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + vresult = self._validate_msg(line, string_idx) + if isinstance(vresult, Err): + return vresult + + # If the string group is wrapped inside an Atom node, we must make sure + # to later replace that Atom with our new (merged) string leaf. + atom_node = LL[string_idx].parent + + # We will place BREAK_MARK in between every two substrings that we + # merge. We will then later go through our final result and use the + # various instances of BREAK_MARK we find to add the right values to + # the custom split map. + BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@" + + QUOTE = LL[string_idx].value[-1] + + def make_naked(string: str, string_prefix: str) -> str: + """Strip @string (i.e. make it a "naked" string) + + Pre-conditions: + * assert_is_leaf_string(@string) + + Returns: + A string that is identical to @string except that + @string_prefix has been stripped, the surrounding QUOTE + characters have been removed, and any remaining QUOTE + characters have been escaped. + """ + assert_is_leaf_string(string) + + RE_EVEN_BACKSLASHES = r"(?:(?= 0 + ), "Logic error while filling the custom string breakpoint cache." + + temp_string = temp_string[mark_idx + len(BREAK_MARK) :] + breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1 + custom_splits.append(CustomSplit(has_prefix, breakpoint_idx)) + + string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, "")) + + if atom_node is not None: + replace_child(atom_node, string_leaf) + + # Build the final line ('new_line') that this method will later return. + new_line = line.clone() + for (i, leaf) in enumerate(LL): + if i == string_idx: + new_line.append(string_leaf) + + if string_idx <= i < string_idx + num_of_strings: + for comment_leaf in line.comments_after(LL[i]): + new_line.append(comment_leaf, preformatted=True) + continue + + append_leaves(new_line, line, [leaf]) + + self.add_custom_splits(string_leaf.value, custom_splits) + return Ok(new_line) + + @staticmethod + def _validate_msg(line: Line, string_idx: int) -> TResult[None]: + """Validate (M)erge (S)tring (G)roup + + Transform-time string validation logic for __merge_string_group(...). + + Returns: + * Ok(None), if ALL validation checks (listed below) pass. + OR + * Err(CannotTransform), if any of the following are true: + - The target string group does not contain ANY stand-alone comments. + - The target string is not in a string group (i.e. it has no + adjacent strings). + - The string group has more than one inline comment. + - The string group has an inline comment that appears to be a pragma. + - The set of all string prefixes in the string group is of + length greater than one and is not equal to {"", "f"}. + - The string group consists of raw strings. + """ + # We first check for "inner" stand-alone comments (i.e. stand-alone + # comments that have a string leaf before them AND after them). + for inc in [1, -1]: + i = string_idx + found_sa_comment = False + is_valid_index = is_valid_index_factory(line.leaves) + while is_valid_index(i) and line.leaves[i].type in [ + token.STRING, + STANDALONE_COMMENT, + ]: + if line.leaves[i].type == STANDALONE_COMMENT: + found_sa_comment = True + elif found_sa_comment: + return TErr( + "StringMerger does NOT merge string groups which contain " + "stand-alone comments." + ) + + i += inc + + num_of_inline_string_comments = 0 + set_of_prefixes = set() + num_of_strings = 0 + for leaf in line.leaves[string_idx:]: + if leaf.type != token.STRING: + # If the string group is trailed by a comma, we count the + # comments trailing the comma to be one of the string group's + # comments. + if leaf.type == token.COMMA and id(leaf) in line.comments: + num_of_inline_string_comments += 1 + break + + if has_triple_quotes(leaf.value): + return TErr("StringMerger does NOT merge multiline strings.") + + num_of_strings += 1 + prefix = get_string_prefix(leaf.value) + if "r" in prefix: + return TErr("StringMerger does NOT merge raw strings.") + + set_of_prefixes.add(prefix) + + if id(leaf) in line.comments: + num_of_inline_string_comments += 1 + if contains_pragma_comment(line.comments[id(leaf)]): + return TErr("Cannot merge strings which have pragma comments.") + + if num_of_strings < 2: + return TErr( + f"Not enough strings to merge (num_of_strings={num_of_strings})." + ) + + if num_of_inline_string_comments > 1: + return TErr( + f"Too many inline string comments ({num_of_inline_string_comments})." + ) + + if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}: + return TErr(f"Too many different prefixes ({set_of_prefixes}).") + + return Ok(None) + + +class StringParenStripper(StringTransformer): + """StringTransformer that strips surrounding parentheses from strings. + + Requirements: + The line contains a string which is surrounded by parentheses and: + - The target string is NOT the only argument to a function call. + - The target string is NOT a "pointless" string. + - If the target string contains a PERCENT, the brackets are not + preceeded or followed by an operator with higher precedence than + PERCENT. + + Transformations: + The parentheses mentioned in the 'Requirements' section are stripped. + + Collaborations: + StringParenStripper has its own inherent usefulness, but it is also + relied on to clean up the parentheses created by StringParenWrapper (in + the event that they are no longer needed). + """ + + def do_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + for (idx, leaf) in enumerate(LL): + # Should be a string... + if leaf.type != token.STRING: + continue + + # If this is a "pointless" string... + if ( + leaf.parent + and leaf.parent.parent + and leaf.parent.parent.type == syms.simple_stmt + ): + continue + + # Should be preceded by a non-empty LPAR... + if ( + not is_valid_index(idx - 1) + or LL[idx - 1].type != token.LPAR + or is_empty_lpar(LL[idx - 1]) + ): + continue + + # That LPAR should NOT be preceded by a function name or a closing + # bracket (which could be a function which returns a function or a + # list/dictionary that contains a function)... + if is_valid_index(idx - 2) and ( + LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS + ): + continue + + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + next_idx = string_parser.parse(LL, string_idx) + + # if the leaves in the parsed string include a PERCENT, we need to + # make sure the initial LPAR is NOT preceded by an operator with + # higher or equal precedence to PERCENT + if is_valid_index(idx - 2): + # mypy can't quite follow unless we name this + before_lpar = LL[idx - 2] + if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and ( + ( + before_lpar.type + in { + token.STAR, + token.AT, + token.SLASH, + token.DOUBLESLASH, + token.PERCENT, + token.TILDE, + token.DOUBLESTAR, + token.AWAIT, + token.LSQB, + token.LPAR, + } + ) + or ( + # only unary PLUS/MINUS + before_lpar.parent + and before_lpar.parent.type == syms.factor + and (before_lpar.type in {token.PLUS, token.MINUS}) + ) + ): + continue + + # Should be followed by a non-empty RPAR... + if ( + is_valid_index(next_idx) + and LL[next_idx].type == token.RPAR + and not is_empty_rpar(LL[next_idx]) + ): + # That RPAR should NOT be followed by anything with higher + # precedence than PERCENT + if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in { + token.DOUBLESTAR, + token.LSQB, + token.LPAR, + token.DOT, + }: + continue + + return Ok(string_idx) + + return TErr("This line has no strings wrapped in parens.") + + def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + LL = line.leaves + + string_parser = StringParser() + rpar_idx = string_parser.parse(LL, string_idx) + + for leaf in (LL[string_idx - 1], LL[rpar_idx]): + if line.comments_after(leaf): + yield TErr( + "Will not strip parentheses which have comments attached to them." + ) + return + + new_line = line.clone() + new_line.comments = line.comments.copy() + try: + append_leaves(new_line, line, LL[: string_idx - 1]) + except BracketMatchError: + # HACK: I believe there is currently a bug somewhere in + # right_hand_split() that is causing brackets to not be tracked + # properly by a shared BracketTracker. + append_leaves(new_line, line, LL[: string_idx - 1], preformatted=True) + + string_leaf = Leaf(token.STRING, LL[string_idx].value) + LL[string_idx - 1].remove() + replace_child(LL[string_idx], string_leaf) + new_line.append(string_leaf) + + append_leaves( + new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :] + ) + + LL[rpar_idx].remove() + + yield Ok(new_line) + + +class BaseStringSplitter(StringTransformer): + """ + Abstract class for StringTransformers which transform a Line's strings by splitting + them or placing them on their own lines where necessary to avoid going over + the configured line length. + + Requirements: + * The target string value is responsible for the line going over the + line length limit. It follows that after all of black's other line + split methods have been exhausted, this line (or one of the resulting + lines after all line splits are performed) would still be over the + line_length limit unless we split this string. + AND + * The target string is NOT a "pointless" string (i.e. a string that has + no parent or siblings). + AND + * The target string is not followed by an inline comment that appears + to be a pragma. + AND + * The target string is not a multiline (i.e. triple-quote) string. + """ + + @abstractmethod + def do_splitter_match(self, line: Line) -> TMatchResult: + """ + BaseStringSplitter asks its clients to override this method instead of + `StringTransformer.do_match(...)`. + + Follows the same protocol as `StringTransformer.do_match(...)`. + + Refer to `help(StringTransformer.do_match)` for more information. + """ + + def do_match(self, line: Line) -> TMatchResult: + match_result = self.do_splitter_match(line) + if isinstance(match_result, Err): + return match_result + + string_idx = match_result.ok() + vresult = self._validate(line, string_idx) + if isinstance(vresult, Err): + return vresult + + return match_result + + def _validate(self, line: Line, string_idx: int) -> TResult[None]: + """ + Checks that @line meets all of the requirements listed in this classes' + docstring. Refer to `help(BaseStringSplitter)` for a detailed + description of those requirements. + + Returns: + * Ok(None), if ALL of the requirements are met. + OR + * Err(CannotTransform), if ANY of the requirements are NOT met. + """ + LL = line.leaves + + string_leaf = LL[string_idx] + + max_string_length = self._get_max_string_length(line, string_idx) + if len(string_leaf.value) <= max_string_length: + return TErr( + "The string itself is not what is causing this line to be too long." + ) + + if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [ + token.STRING, + token.NEWLINE, + ]: + return TErr( + f"This string ({string_leaf.value}) appears to be pointless (i.e. has" + " no parent)." + ) + + if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( + line.comments[id(line.leaves[string_idx])] + ): + return TErr( + "Line appears to end with an inline pragma comment. Splitting the line" + " could modify the pragma's behavior." + ) + + if has_triple_quotes(string_leaf.value): + return TErr("We cannot split multiline strings.") + + return Ok(None) + + def _get_max_string_length(self, line: Line, string_idx: int) -> int: + """ + Calculates the max string length used when attempting to determine + whether or not the target string is responsible for causing the line to + go over the line length limit. + + WARNING: This method is tightly coupled to both StringSplitter and + (especially) StringParenWrapper. There is probably a better way to + accomplish what is being done here. + + Returns: + max_string_length: such that `line.leaves[string_idx].value > + max_string_length` implies that the target string IS responsible + for causing this line to exceed the line length limit. + """ + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + # We use the shorthand "WMA4" in comments to abbreviate "We must + # account for". When giving examples, we use STRING to mean some/any + # valid string. + # + # Finally, we use the following convenience variables: + # + # P: The leaf that is before the target string leaf. + # N: The leaf that is after the target string leaf. + # NN: The leaf that is after N. + + # WMA4 the whitespace at the beginning of the line. + offset = line.depth * 4 + + if is_valid_index(string_idx - 1): + p_idx = string_idx - 1 + if ( + LL[string_idx - 1].type == token.LPAR + and LL[string_idx - 1].value == "" + and string_idx >= 2 + ): + # If the previous leaf is an empty LPAR placeholder, we should skip it. + p_idx -= 1 + + P = LL[p_idx] + if P.type == token.PLUS: + # WMA4 a space and a '+' character (e.g. `+ STRING`). + offset += 2 + + if P.type == token.COMMA: + # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`]. + offset += 3 + + if P.type in [token.COLON, token.EQUAL, token.NAME]: + # This conditional branch is meant to handle dictionary keys, + # variable assignments, 'return STRING' statement lines, and + # 'else STRING' ternary expression lines. + + # WMA4 a single space. + offset += 1 + + # WMA4 the lengths of any leaves that came before that space, + # but after any closing bracket before that space. + for leaf in reversed(LL[: p_idx + 1]): + offset += len(str(leaf)) + if leaf.type in CLOSING_BRACKETS: + break + + if is_valid_index(string_idx + 1): + N = LL[string_idx + 1] + if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2: + # If the next leaf is an empty RPAR placeholder, we should skip it. + N = LL[string_idx + 2] + + if N.type == token.COMMA: + # WMA4 a single comma at the end of the string (e.g `STRING,`). + offset += 1 + + if is_valid_index(string_idx + 2): + NN = LL[string_idx + 2] + + if N.type == token.DOT and NN.type == token.NAME: + # This conditional branch is meant to handle method calls invoked + # off of a string literal up to and including the LPAR character. + + # WMA4 the '.' character. + offset += 1 + + if ( + is_valid_index(string_idx + 3) + and LL[string_idx + 3].type == token.LPAR + ): + # WMA4 the left parenthesis character. + offset += 1 + + # WMA4 the length of the method's name. + offset += len(NN.value) + + has_comments = False + for comment_leaf in line.comments_after(LL[string_idx]): + if not has_comments: + has_comments = True + # WMA4 two spaces before the '#' character. + offset += 2 + + # WMA4 the length of the inline comment. + offset += len(comment_leaf.value) + + max_string_length = self.line_length - offset + return max_string_length + + +class StringSplitter(CustomSplitMapMixin, BaseStringSplitter): + """ + StringTransformer that splits "atom" strings (i.e. strings which exist on + lines by themselves). + + Requirements: + * The line consists ONLY of a single string (with the exception of a + '+' symbol which MAY exist at the start of the line), MAYBE a string + trailer, and MAYBE a trailing comma. + AND + * All of the requirements listed in BaseStringSplitter's docstring. + + Transformations: + The string mentioned in the 'Requirements' section is split into as + many substrings as necessary to adhere to the configured line length. + + In the final set of substrings, no substring should be smaller than + MIN_SUBSTR_SIZE characters. + + The string will ONLY be split on spaces (i.e. each new substring should + start with a space). Note that the string will NOT be split on a space + which is escaped with a backslash. + + If the string is an f-string, it will NOT be split in the middle of an + f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x + else bar()} is an f-expression). + + If the string that is being split has an associated set of custom split + records and those custom splits will NOT result in any line going over + the configured line length, those custom splits are used. Otherwise the + string is split as late as possible (from left-to-right) while still + adhering to the transformation rules listed above. + + Collaborations: + StringSplitter relies on StringMerger to construct the appropriate + CustomSplit objects and add them to the custom split map. + """ + + MIN_SUBSTR_SIZE = 6 + # Matches an "f-expression" (e.g. {var}) that might be found in an f-string. + RE_FEXPR = r""" + (? TMatchResult: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + + idx = 0 + + # The first leaf MAY be a '+' symbol... + if is_valid_index(idx) and LL[idx].type == token.PLUS: + idx += 1 + + # The next/first leaf MAY be an empty LPAR... + if is_valid_index(idx) and is_empty_lpar(LL[idx]): + idx += 1 + + # The next/first leaf MUST be a string... + if not is_valid_index(idx) or LL[idx].type != token.STRING: + return TErr("Line does not start with a string.") + + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # That string MAY be followed by an empty RPAR... + if is_valid_index(idx) and is_empty_rpar(LL[idx]): + idx += 1 + + # That string / empty RPAR leaf MAY be followed by a comma... + if is_valid_index(idx) and LL[idx].type == token.COMMA: + idx += 1 + + # But no more leaves are allowed... + if is_valid_index(idx): + return TErr("This line does not end with a string.") + + return Ok(string_idx) + + def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + LL = line.leaves + + QUOTE = LL[string_idx].value[-1] + + is_valid_index = is_valid_index_factory(LL) + insert_str_child = insert_str_child_factory(LL[string_idx]) + + prefix = get_string_prefix(LL[string_idx].value) + + # We MAY choose to drop the 'f' prefix from substrings that don't + # contain any f-expressions, but ONLY if the original f-string + # contains at least one f-expression. Otherwise, we will alter the AST + # of the program. + drop_pointless_f_prefix = ("f" in prefix) and re.search( + self.RE_FEXPR, LL[string_idx].value, re.VERBOSE + ) + + first_string_line = True + starts_with_plus = LL[0].type == token.PLUS + + def line_needs_plus() -> bool: + return first_string_line and starts_with_plus + + def maybe_append_plus(new_line: Line) -> None: + """ + Side Effects: + If @line starts with a plus and this is the first line we are + constructing, this function appends a PLUS leaf to @new_line + and replaces the old PLUS leaf in the node structure. Otherwise + this function does nothing. + """ + if line_needs_plus(): + plus_leaf = Leaf(token.PLUS, "+") + replace_child(LL[0], plus_leaf) + new_line.append(plus_leaf) + + ends_with_comma = ( + is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA + ) + + def max_last_string() -> int: + """ + Returns: + The max allowed length of the string value used for the last + line we will construct. + """ + result = self.line_length + result -= line.depth * 4 + result -= 1 if ends_with_comma else 0 + result -= 2 if line_needs_plus() else 0 + return result + + # --- Calculate Max Break Index (for string value) + # We start with the line length limit + max_break_idx = self.line_length + # The last index of a string of length N is N-1. + max_break_idx -= 1 + # Leading whitespace is not present in the string value (e.g. Leaf.value). + max_break_idx -= line.depth * 4 + if max_break_idx < 0: + yield TErr( + f"Unable to split {LL[string_idx].value} at such high of a line depth:" + f" {line.depth}" + ) + return + + # Check if StringMerger registered any custom splits. + custom_splits = self.pop_custom_splits(LL[string_idx].value) + # We use them ONLY if none of them would produce lines that exceed the + # line limit. + use_custom_breakpoints = bool( + custom_splits + and all(csplit.break_idx <= max_break_idx for csplit in custom_splits) + ) + + # Temporary storage for the remaining chunk of the string line that + # can't fit onto the line currently being constructed. + rest_value = LL[string_idx].value + + def more_splits_should_be_made() -> bool: + """ + Returns: + True iff `rest_value` (the remaining string value from the last + split), should be split again. + """ + if use_custom_breakpoints: + return len(custom_splits) > 1 + else: + return len(rest_value) > max_last_string() + + string_line_results: List[Ok[Line]] = [] + while more_splits_should_be_made(): + if use_custom_breakpoints: + # Custom User Split (manual) + csplit = custom_splits.pop(0) + break_idx = csplit.break_idx + else: + # Algorithmic Split (automatic) + max_bidx = max_break_idx - 2 if line_needs_plus() else max_break_idx + maybe_break_idx = self._get_break_idx(rest_value, max_bidx) + if maybe_break_idx is None: + # If we are unable to algorithmically determine a good split + # and this string has custom splits registered to it, we + # fall back to using them--which means we have to start + # over from the beginning. + if custom_splits: + rest_value = LL[string_idx].value + string_line_results = [] + first_string_line = True + use_custom_breakpoints = True + continue + + # Otherwise, we stop splitting here. + break + + break_idx = maybe_break_idx + + # --- Construct `next_value` + next_value = rest_value[:break_idx] + QUOTE + if ( + # Are we allowed to try to drop a pointless 'f' prefix? + drop_pointless_f_prefix + # If we are, will we be successful? + and next_value != self._normalize_f_string(next_value, prefix) + ): + # If the current custom split did NOT originally use a prefix, + # then `csplit.break_idx` will be off by one after removing + # the 'f' prefix. + break_idx = ( + break_idx + 1 + if use_custom_breakpoints and not csplit.has_prefix + else break_idx + ) + next_value = rest_value[:break_idx] + QUOTE + next_value = self._normalize_f_string(next_value, prefix) + + # --- Construct `next_leaf` + next_leaf = Leaf(token.STRING, next_value) + insert_str_child(next_leaf) + self._maybe_normalize_string_quotes(next_leaf) + + # --- Construct `next_line` + next_line = line.clone() + maybe_append_plus(next_line) + next_line.append(next_leaf) + string_line_results.append(Ok(next_line)) + + rest_value = prefix + QUOTE + rest_value[break_idx:] + first_string_line = False + + yield from string_line_results + + if drop_pointless_f_prefix: + rest_value = self._normalize_f_string(rest_value, prefix) + + rest_leaf = Leaf(token.STRING, rest_value) + insert_str_child(rest_leaf) + + # NOTE: I could not find a test case that verifies that the following + # line is actually necessary, but it seems to be. Otherwise we risk + # not normalizing the last substring, right? + self._maybe_normalize_string_quotes(rest_leaf) + + last_line = line.clone() + maybe_append_plus(last_line) + + # If there are any leaves to the right of the target string... + if is_valid_index(string_idx + 1): + # We use `temp_value` here to determine how long the last line + # would be if we were to append all the leaves to the right of the + # target string to the last string line. + temp_value = rest_value + for leaf in LL[string_idx + 1 :]: + temp_value += str(leaf) + if leaf.type == token.LPAR: + break + + # Try to fit them all on the same line with the last substring... + if ( + len(temp_value) <= max_last_string() + or LL[string_idx + 1].type == token.COMMA + ): + last_line.append(rest_leaf) + append_leaves(last_line, line, LL[string_idx + 1 :]) + yield Ok(last_line) + # Otherwise, place the last substring on one line and everything + # else on a line below that... + else: + last_line.append(rest_leaf) + yield Ok(last_line) + + non_string_line = line.clone() + append_leaves(non_string_line, line, LL[string_idx + 1 :]) + yield Ok(non_string_line) + # Else the target string was the last leaf... + else: + last_line.append(rest_leaf) + last_line.comments = line.comments.copy() + yield Ok(last_line) + + def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]: + """ + This method contains the algorithm that StringSplitter uses to + determine which character to split each string at. + + Args: + @string: The substring that we are attempting to split. + @max_break_idx: The ideal break index. We will return this value if it + meets all the necessary conditions. In the likely event that it + doesn't we will try to find the closest index BELOW @max_break_idx + that does. If that fails, we will expand our search by also + considering all valid indices ABOVE @max_break_idx. + + Pre-Conditions: + * assert_is_leaf_string(@string) + * 0 <= @max_break_idx < len(@string) + + Returns: + break_idx, if an index is able to be found that meets all of the + conditions listed in the 'Transformations' section of this classes' + docstring. + OR + None, otherwise. + """ + is_valid_index = is_valid_index_factory(string) + + assert is_valid_index(max_break_idx) + assert_is_leaf_string(string) + + _fexpr_slices: Optional[List[Tuple[Index, Index]]] = None + + def fexpr_slices() -> Iterator[Tuple[Index, Index]]: + """ + Yields: + All ranges of @string which, if @string were to be split there, + would result in the splitting of an f-expression (which is NOT + allowed). + """ + nonlocal _fexpr_slices + + if _fexpr_slices is None: + _fexpr_slices = [] + for match in re.finditer(self.RE_FEXPR, string, re.VERBOSE): + _fexpr_slices.append(match.span()) + + yield from _fexpr_slices + + is_fstring = "f" in get_string_prefix(string) + + def breaks_fstring_expression(i: Index) -> bool: + """ + Returns: + True iff returning @i would result in the splitting of an + f-expression (which is NOT allowed). + """ + if not is_fstring: + return False + + for (start, end) in fexpr_slices(): + if start <= i < end: + return True + + return False + + def passes_all_checks(i: Index) -> bool: + """ + Returns: + True iff ALL of the conditions listed in the 'Transformations' + section of this classes' docstring would be be met by returning @i. + """ + is_space = string[i] == " " + + is_not_escaped = True + j = i - 1 + while is_valid_index(j) and string[j] == "\\": + is_not_escaped = not is_not_escaped + j -= 1 + + is_big_enough = ( + len(string[i:]) >= self.MIN_SUBSTR_SIZE + and len(string[:i]) >= self.MIN_SUBSTR_SIZE + ) + return ( + is_space + and is_not_escaped + and is_big_enough + and not breaks_fstring_expression(i) + ) + + # First, we check all indices BELOW @max_break_idx. + break_idx = max_break_idx + while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx): + break_idx -= 1 + + if not passes_all_checks(break_idx): + # If that fails, we check all indices ABOVE @max_break_idx. + # + # If we are able to find a valid index here, the next line is going + # to be longer than the specified line length, but it's probably + # better than doing nothing at all. + break_idx = max_break_idx + 1 + while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx): + break_idx += 1 + + if not is_valid_index(break_idx) or not passes_all_checks(break_idx): + return None + + return break_idx + + def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None: + if self.normalize_strings: + leaf.value = normalize_string_quotes(leaf.value) + + def _normalize_f_string(self, string: str, prefix: str) -> str: + """ + Pre-Conditions: + * assert_is_leaf_string(@string) + + Returns: + * If @string is an f-string that contains no f-expressions, we + return a string identical to @string except that the 'f' prefix + has been stripped and all double braces (i.e. '{{' or '}}') have + been normalized (i.e. turned into '{' or '}'). + OR + * Otherwise, we return @string. + """ + assert_is_leaf_string(string) + + if "f" in prefix and not re.search(self.RE_FEXPR, string, re.VERBOSE): + new_prefix = prefix.replace("f", "") + + temp = string[len(prefix) :] + temp = re.sub(r"\{\{", "{", temp) + temp = re.sub(r"\}\}", "}", temp) + new_string = temp + + return f"{new_prefix}{new_string}" + else: + return string + + +class StringParenWrapper(CustomSplitMapMixin, BaseStringSplitter): + """ + StringTransformer that splits non-"atom" strings (i.e. strings that do not + exist on lines by themselves). + + Requirements: + All of the requirements listed in BaseStringSplitter's docstring in + addition to the requirements listed below: + + * The line is a return/yield statement, which returns/yields a string. + OR + * The line is part of a ternary expression (e.g. `x = y if cond else + z`) such that the line starts with `else `, where is + some string. + OR + * The line is an assert statement, which ends with a string. + OR + * The line is an assignment statement (e.g. `x = ` or `x += + `) such that the variable is being assigned the value of some + string. + OR + * The line is a dictionary key assignment where some valid key is being + assigned the value of some string. + + Transformations: + The chosen string is wrapped in parentheses and then split at the LPAR. + + We then have one line which ends with an LPAR and another line that + starts with the chosen string. The latter line is then split again at + the RPAR. This results in the RPAR (and possibly a trailing comma) + being placed on its own line. + + NOTE: If any leaves exist to the right of the chosen string (except + for a trailing comma, which would be placed after the RPAR), those + leaves are placed inside the parentheses. In effect, the chosen + string is not necessarily being "wrapped" by parentheses. We can, + however, count on the LPAR being placed directly before the chosen + string. + + In other words, StringParenWrapper creates "atom" strings. These + can then be split again by StringSplitter, if necessary. + + Collaborations: + In the event that a string line split by StringParenWrapper is + changed such that it no longer needs to be given its own line, + StringParenWrapper relies on StringParenStripper to clean up the + parentheses it created. + """ + + def do_splitter_match(self, line: Line) -> TMatchResult: + LL = line.leaves + + string_idx = ( + self._return_match(LL) + or self._else_match(LL) + or self._assert_match(LL) + or self._assign_match(LL) + or self._dict_match(LL) + ) + + if string_idx is not None: + string_value = line.leaves[string_idx].value + # If the string has no spaces... + if " " not in string_value: + # And will still violate the line length limit when split... + max_string_length = self.line_length - ((line.depth + 1) * 4) + if len(string_value) > max_string_length: + # And has no associated custom splits... + if not self.has_custom_splits(string_value): + # Then we should NOT put this string on its own line. + return TErr( + "We do not wrap long strings in parentheses when the" + " resultant line would still be over the specified line" + " length and can't be split further by StringSplitter." + ) + return Ok(string_idx) + + return TErr("This line does not contain any non-atomic strings.") + + @staticmethod + def _return_match(LL: List[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the return/yield statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is apart of a return/yield statement and the first leaf + # contains either the "return" or "yield" keywords... + if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[ + 0 + ].value in ["return", "yield"]: + is_valid_index = is_valid_index_factory(LL) + + idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 + # The next visible leaf MUST contain a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + return idx + + return None + + @staticmethod + def _else_match(LL: List[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the ternary expression + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is apart of a ternary expression and the first leaf + # contains the "else" keyword... + if ( + parent_type(LL[0]) == syms.test + and LL[0].type == token.NAME + and LL[0].value == "else" + ): + is_valid_index = is_valid_index_factory(LL) + + idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1 + # The next visible leaf MUST contain a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + return idx + + return None + + @staticmethod + def _assert_match(LL: List[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the assert statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is apart of an assert statement and the first leaf + # contains the "assert" keyword... + if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert": + is_valid_index = is_valid_index_factory(LL) + + for (i, leaf) in enumerate(LL): + # We MUST find a comma... + if leaf.type == token.COMMA: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That comma MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + @staticmethod + def _assign_match(LL: List[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the assignment statement + requirements listed in the 'Requirements' section of this classes' + docstring. + OR + None, otherwise. + """ + # If this line is apart of an expression statement or is a function + # argument AND the first leaf contains a variable name... + if ( + parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power] + and LL[0].type == token.NAME + ): + is_valid_index = is_valid_index_factory(LL) + + for (i, leaf) in enumerate(LL): + # We MUST find either an '=' or '+=' symbol... + if leaf.type in [token.EQUAL, token.PLUSEQUAL]: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That symbol MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # The next leaf MAY be a comma iff this line is apart + # of a function argument... + if ( + parent_type(LL[0]) == syms.argument + and is_valid_index(idx) + and LL[idx].type == token.COMMA + ): + idx += 1 + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + @staticmethod + def _dict_match(LL: List[Leaf]) -> Optional[int]: + """ + Returns: + string_idx such that @LL[string_idx] is equal to our target (i.e. + matched) string, if this line matches the dictionary key assignment + statement requirements listed in the 'Requirements' section of this + classes' docstring. + OR + None, otherwise. + """ + # If this line is apart of a dictionary key assignment... + if syms.dictsetmaker in [parent_type(LL[0]), parent_type(LL[0].parent)]: + is_valid_index = is_valid_index_factory(LL) + + for (i, leaf) in enumerate(LL): + # We MUST find a colon... + if leaf.type == token.COLON: + idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1 + + # That colon MUST be followed by a string... + if is_valid_index(idx) and LL[idx].type == token.STRING: + string_idx = idx + + # Skip the string trailer, if one exists. + string_parser = StringParser() + idx = string_parser.parse(LL, string_idx) + + # That string MAY be followed by a comma... + if is_valid_index(idx) and LL[idx].type == token.COMMA: + idx += 1 + + # But no more leaves are allowed... + if not is_valid_index(idx): + return string_idx + + return None + + def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]: + LL = line.leaves + + is_valid_index = is_valid_index_factory(LL) + insert_str_child = insert_str_child_factory(LL[string_idx]) + + comma_idx = -1 + ends_with_comma = False + if LL[comma_idx].type == token.COMMA: + ends_with_comma = True + + leaves_to_steal_comments_from = [LL[string_idx]] + if ends_with_comma: + leaves_to_steal_comments_from.append(LL[comma_idx]) + + # --- First Line + first_line = line.clone() + left_leaves = LL[:string_idx] + + # We have to remember to account for (possibly invisible) LPAR and RPAR + # leaves that already wrapped the target string. If these leaves do + # exist, we will replace them with our own LPAR and RPAR leaves. + old_parens_exist = False + if left_leaves and left_leaves[-1].type == token.LPAR: + old_parens_exist = True + leaves_to_steal_comments_from.append(left_leaves[-1]) + left_leaves.pop() + + append_leaves(first_line, line, left_leaves) + + lpar_leaf = Leaf(token.LPAR, "(") + if old_parens_exist: + replace_child(LL[string_idx - 1], lpar_leaf) + else: + insert_str_child(lpar_leaf) + first_line.append(lpar_leaf) + + # We throw inline comments that were originally to the right of the + # target string to the top line. They will now be shown to the right of + # the LPAR. + for leaf in leaves_to_steal_comments_from: + for comment_leaf in line.comments_after(leaf): + first_line.append(comment_leaf, preformatted=True) + + yield Ok(first_line) + + # --- Middle (String) Line + # We only need to yield one (possibly too long) string line, since the + # `StringSplitter` will break it down further if necessary. + string_value = LL[string_idx].value + string_line = Line( + mode=line.mode, + depth=line.depth + 1, + inside_brackets=True, + should_split_rhs=line.should_split_rhs, + magic_trailing_comma=line.magic_trailing_comma, + ) + string_leaf = Leaf(token.STRING, string_value) + insert_str_child(string_leaf) + string_line.append(string_leaf) + + old_rpar_leaf = None + if is_valid_index(string_idx + 1): + right_leaves = LL[string_idx + 1 :] + if ends_with_comma: + right_leaves.pop() + + if old_parens_exist: + assert ( + right_leaves and right_leaves[-1].type == token.RPAR + ), "Apparently, old parentheses do NOT exist?!" + old_rpar_leaf = right_leaves.pop() + + append_leaves(string_line, line, right_leaves) + + yield Ok(string_line) + + # --- Last Line + last_line = line.clone() + last_line.bracket_tracker = first_line.bracket_tracker + + new_rpar_leaf = Leaf(token.RPAR, ")") + if old_rpar_leaf is not None: + replace_child(old_rpar_leaf, new_rpar_leaf) + else: + insert_str_child(new_rpar_leaf) + last_line.append(new_rpar_leaf) + + # If the target string ended with a comma, we place this comma to the + # right of the RPAR on the last line. + if ends_with_comma: + comma_leaf = Leaf(token.COMMA, ",") + replace_child(LL[comma_idx], comma_leaf) + last_line.append(comma_leaf) + + yield Ok(last_line) + + +class StringParser: + """ + A state machine that aids in parsing a string's "trailer", which can be + either non-existent, an old-style formatting sequence (e.g. `% varX` or `% + (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX, + varY)`). + + NOTE: A new StringParser object MUST be instantiated for each string + trailer we need to parse. + + Examples: + We shall assume that `line` equals the `Line` object that corresponds + to the following line of python code: + ``` + x = "Some {}.".format("String") + some_other_string + ``` + + Furthermore, we will assume that `string_idx` is some index such that: + ``` + assert line.leaves[string_idx].value == "Some {}." + ``` + + The following code snippet then holds: + ``` + string_parser = StringParser() + idx = string_parser.parse(line.leaves, string_idx) + assert line.leaves[idx].type == token.PLUS + ``` + """ + + DEFAULT_TOKEN = -1 + + # String Parser States + START = 1 + DOT = 2 + NAME = 3 + PERCENT = 4 + SINGLE_FMT_ARG = 5 + LPAR = 6 + RPAR = 7 + DONE = 8 + + # Lookup Table for Next State + _goto: Dict[Tuple[ParserState, NodeType], ParserState] = { + # A string trailer may start with '.' OR '%'. + (START, token.DOT): DOT, + (START, token.PERCENT): PERCENT, + (START, DEFAULT_TOKEN): DONE, + # A '.' MUST be followed by an attribute or method name. + (DOT, token.NAME): NAME, + # A method name MUST be followed by an '(', whereas an attribute name + # is the last symbol in the string trailer. + (NAME, token.LPAR): LPAR, + (NAME, DEFAULT_TOKEN): DONE, + # A '%' symbol can be followed by an '(' or a single argument (e.g. a + # string or variable name). + (PERCENT, token.LPAR): LPAR, + (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG, + # If a '%' symbol is followed by a single argument, that argument is + # the last leaf in the string trailer. + (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE, + # If present, a ')' symbol is the last symbol in a string trailer. + # (NOTE: LPARS and nested RPARS are not included in this lookup table, + # since they are treated as a special case by the parsing logic in this + # classes' implementation.) + (RPAR, DEFAULT_TOKEN): DONE, + } + + def __init__(self) -> None: + self._state = self.START + self._unmatched_lpars = 0 + + def parse(self, leaves: List[Leaf], string_idx: int) -> int: + """ + Pre-conditions: + * @leaves[@string_idx].type == token.STRING + + Returns: + The index directly after the last leaf which is apart of the string + trailer, if a "trailer" exists. + OR + @string_idx + 1, if no string "trailer" exists. + """ + assert leaves[string_idx].type == token.STRING + + idx = string_idx + 1 + while idx < len(leaves) and self._next_state(leaves[idx]): + idx += 1 + return idx + + def _next_state(self, leaf: Leaf) -> bool: + """ + Pre-conditions: + * On the first call to this function, @leaf MUST be the leaf that + was directly after the string leaf in question (e.g. if our target + string is `line.leaves[i]` then the first call to this method must + be `line.leaves[i + 1]`). + * On the next call to this function, the leaf parameter passed in + MUST be the leaf directly following @leaf. + + Returns: + True iff @leaf is apart of the string's trailer. + """ + # We ignore empty LPAR or RPAR leaves. + if is_empty_par(leaf): + return True + + next_token = leaf.type + if next_token == token.LPAR: + self._unmatched_lpars += 1 + + current_state = self._state + + # The LPAR parser state is a special case. We will return True until we + # find the matching RPAR token. + if current_state == self.LPAR: + if next_token == token.RPAR: + self._unmatched_lpars -= 1 + if self._unmatched_lpars == 0: + self._state = self.RPAR + # Otherwise, we use a lookup table to determine the next state. + else: + # If the lookup table matches the current state to the next + # token, we use the lookup table. + if (current_state, next_token) in self._goto: + self._state = self._goto[current_state, next_token] + else: + # Otherwise, we check if a the current state was assigned a + # default. + if (current_state, self.DEFAULT_TOKEN) in self._goto: + self._state = self._goto[current_state, self.DEFAULT_TOKEN] + # If no default has been assigned, then this parser has a logic + # error. + else: + raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!") + + if self._state == self.DONE: + return False + + return True + + +def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]: + """ + Factory for a convenience function that is used to orphan @string_leaf + and then insert multiple new leaves into the same part of the node + structure that @string_leaf had originally occupied. + + Examples: + Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N = + string_leaf.parent`. Assume the node `N` has the following + original structure: + + Node( + expr_stmt, [ + Leaf(NAME, 'x'), + Leaf(EQUAL, '='), + Leaf(STRING, '"foo"'), + ] + ) + + We then run the code snippet shown below. + ``` + insert_str_child = insert_str_child_factory(string_leaf) + + lpar = Leaf(token.LPAR, '(') + insert_str_child(lpar) + + bar = Leaf(token.STRING, '"bar"') + insert_str_child(bar) + + rpar = Leaf(token.RPAR, ')') + insert_str_child(rpar) + ``` + + After which point, it follows that `string_leaf.parent is None` and + the node `N` now has the following structure: + + Node( + expr_stmt, [ + Leaf(NAME, 'x'), + Leaf(EQUAL, '='), + Leaf(LPAR, '('), + Leaf(STRING, '"bar"'), + Leaf(RPAR, ')'), + ] + ) + """ + string_parent = string_leaf.parent + string_child_idx = string_leaf.remove() + + def insert_str_child(child: LN) -> None: + nonlocal string_child_idx + + assert string_parent is not None + assert string_child_idx is not None + + string_parent.insert_child(string_child_idx, child) + string_child_idx += 1 + + return insert_str_child + + +def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]: + """ + Examples: + ``` + my_list = [1, 2, 3] + + is_valid_index = is_valid_index_factory(my_list) + + assert is_valid_index(0) + assert is_valid_index(2) + + assert not is_valid_index(3) + assert not is_valid_index(-1) + ``` + """ + + def is_valid_index(idx: int) -> bool: + """ + Returns: + True iff @idx is positive AND seq[@idx] does NOT raise an + IndexError. + """ + return 0 <= idx < len(seq) + + return is_valid_index diff --git a/tests/test_black.py b/tests/test_black.py index b8e526a953e..29a0731dd9d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -34,6 +34,10 @@ import black from black import Feature, TargetVersion +from black.cache import get_cache_file +from black.debug import DebugVisitor +from black.report import Report +import black.files from pathspec import PathSpec @@ -69,7 +73,7 @@ def cache_dir(exists: bool = True) -> Iterator[Path]: cache_dir = Path(workspace) if not exists: cache_dir = cache_dir / "new" - with patch("black.CACHE_DIR", cache_dir): + with patch("black.cache.CACHE_DIR", cache_dir): yield cache_dir @@ -582,7 +586,7 @@ def test_tab_comment_indentation(self) -> None: self.assertFormatEqual(contents_spc, fs(contents_tab)) def test_report_verbose(self) -> None: - report = black.Report(verbose=True) + report = Report(verbose=True) out_lines = [] err_lines = [] @@ -592,7 +596,7 @@ def out(msg: str, **kwargs: Any) -> None: def err(msg: str, **kwargs: Any) -> None: err_lines.append(msg) - with patch("black.out", out), patch("black.err", err): + with patch("black.output._out", out), patch("black.output._err", err): report.done(Path("f1"), black.Changed.NO) self.assertEqual(len(out_lines), 1) self.assertEqual(len(err_lines), 0) @@ -684,7 +688,7 @@ def err(msg: str, **kwargs: Any) -> None: ) def test_report_quiet(self) -> None: - report = black.Report(quiet=True) + report = Report(quiet=True) out_lines = [] err_lines = [] @@ -694,7 +698,7 @@ def out(msg: str, **kwargs: Any) -> None: def err(msg: str, **kwargs: Any) -> None: err_lines.append(msg) - with patch("black.out", out), patch("black.err", err): + with patch("black.output._out", out), patch("black.output._err", err): report.done(Path("f1"), black.Changed.NO) self.assertEqual(len(out_lines), 0) self.assertEqual(len(err_lines), 0) @@ -788,7 +792,7 @@ def out(msg: str, **kwargs: Any) -> None: def err(msg: str, **kwargs: Any) -> None: err_lines.append(msg) - with patch("black.out", out), patch("black.err", err): + with patch("black.output._out", out), patch("black.output._err", err): report.done(Path("f1"), black.Changed.NO) self.assertEqual(len(out_lines), 0) self.assertEqual(len(err_lines), 0) @@ -1005,8 +1009,8 @@ def out(msg: str, **kwargs: Any) -> None: def err(msg: str, **kwargs: Any) -> None: err_lines.append(msg) - with patch("black.out", out), patch("black.err", err): - black.DebugVisitor.show(source) + with patch("black.debug.out", out): + DebugVisitor.show(source) actual = "\n".join(out_lines) + "\n" log_name = "" if expected != actual: @@ -1054,7 +1058,7 @@ def out(msg: str, **kwargs: Any) -> None: def err(msg: str, **kwargs: Any) -> None: err_lines.append(msg) - with patch("black.out", out), patch("black.err", err): + with patch("black.output._out", out), patch("black.output._err", err): with self.assertRaises(AssertionError): self.assertFormatEqual("j = [1, 2, 3]", "j = [1, 2, 3,]") @@ -1066,7 +1070,7 @@ def err(msg: str, **kwargs: Any) -> None: def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: - cache_file = black.get_cache_file(mode) + cache_file = get_cache_file(mode) with cache_file.open("w") as fobj: fobj.write("this is not a pickle") self.assertEqual(black.read_cache(mode), {}) @@ -1120,7 +1124,7 @@ def test_no_cache_when_writeback_diff(self) -> None: "black.write_cache" ) as write_cache: self.invokeBlack([str(src), "--diff"]) - cache_file = black.get_cache_file(mode) + cache_file = get_cache_file(mode) self.assertFalse(cache_file.exists()) write_cache.assert_not_called() read_cache.assert_not_called() @@ -1135,7 +1139,7 @@ def test_no_cache_when_writeback_color_diff(self) -> None: "black.write_cache" ) as write_cache: self.invokeBlack([str(src), "--diff", "--color"]) - cache_file = black.get_cache_file(mode) + cache_file = get_cache_file(mode) self.assertFalse(cache_file.exists()) write_cache.assert_not_called() read_cache.assert_not_called() @@ -1173,7 +1177,7 @@ def test_no_cache_when_stdin(self) -> None: black.main, ["-"], input=BytesIO(b"print('hello')") ) self.assertEqual(result.exit_code, 0) - cache_file = black.get_cache_file(mode) + cache_file = get_cache_file(mode) self.assertFalse(cache_file.exists()) def test_read_cache_no_cachefile(self) -> None: @@ -1960,7 +1964,10 @@ def test_find_project_root(self) -> None: self.assertEqual(black.find_project_root((src_dir,)), src_dir.resolve()) self.assertEqual(black.find_project_root((src_python,)), src_dir.resolve()) - @patch("black.find_user_pyproject_toml", black.find_user_pyproject_toml.__wrapped__) + @patch( + "black.files.find_user_pyproject_toml", + black.files.find_user_pyproject_toml.__wrapped__, + ) def test_find_user_pyproject_toml_linux(self) -> None: if system() == "Windows": return @@ -1970,7 +1977,7 @@ def test_find_user_pyproject_toml_linux(self) -> None: tmp_user_config = Path(workspace) / "black" with patch.dict("os.environ", {"XDG_CONFIG_HOME": workspace}): self.assertEqual( - black.find_user_pyproject_toml(), tmp_user_config.resolve() + black.files.find_user_pyproject_toml(), tmp_user_config.resolve() ) # Test fallback for XDG_CONFIG_HOME @@ -1978,7 +1985,7 @@ def test_find_user_pyproject_toml_linux(self) -> None: os.environ.pop("XDG_CONFIG_HOME", None) fallback_user_config = Path("~/.config").expanduser() / "black" self.assertEqual( - black.find_user_pyproject_toml(), fallback_user_config.resolve() + black.files.find_user_pyproject_toml(), fallback_user_config.resolve() ) def test_find_user_pyproject_toml_windows(self) -> None: @@ -1986,7 +1993,9 @@ def test_find_user_pyproject_toml_windows(self) -> None: return user_config_path = Path.home() / ".black" - self.assertEqual(black.find_user_pyproject_toml(), user_config_path.resolve()) + self.assertEqual( + black.files.find_user_pyproject_toml(), user_config_path.resolve() + ) def test_bpo_33660_workaround(self) -> None: if system() == "Windows": diff --git a/tests/test_format.py b/tests/test_format.py index 78f2b558a4f..e59d5218d2e 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -71,10 +71,27 @@ SOURCES = [ - "tests/test_black.py", - "tests/test_format.py", - "tests/test_blackd.py", "src/black/__init__.py", + "src/black/__main__.py", + "src/black/brackets.py", + "src/black/cache.py", + "src/black/comments.py", + "src/black/concurrency.py", + "src/black/const.py", + "src/black/debug.py", + "src/black/files.py", + "src/black/linegen.py", + "src/black/lines.py", + "src/black/mode.py", + "src/black/nodes.py", + "src/black/numerics.py", + "src/black/output.py", + "src/black/parsing.py", + "src/black/report.py", + "src/black/rusty.py", + "src/black/strings.py", + "src/black/trans.py", + "src/blackd/__init__.py", "src/blib2to3/pygram.py", "src/blib2to3/pytree.py", "src/blib2to3/pgen2/conv.py", @@ -86,6 +103,13 @@ "src/blib2to3/pgen2/tokenize.py", "src/blib2to3/pgen2/token.py", "setup.py", + "tests/test_black.py", + "tests/test_blackd.py", + "tests/test_format.py", + "tests/test_primer.py", + "tests/optional.py", + "tests/util.py", + "tests/conftest.py", ] diff --git a/tests/util.py b/tests/util.py index 3670952ba8c..1e86a3f3d78 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,9 +2,12 @@ import unittest from pathlib import Path from typing import List, Tuple, Any -import black from functools import partial +import black +from black.output import out, err +from black.debug import DebugVisitor + THIS_DIR = Path(__file__).parent PROJECT_ROOT = THIS_DIR.parent EMPTY_LINE = "# EMPTY LINE WITH WHITESPACE" + " (this comment will be removed)" @@ -26,21 +29,21 @@ class BlackBaseTestCase(unittest.TestCase): def assertFormatEqual(self, expected: str, actual: str) -> None: if actual != expected and not os.environ.get("SKIP_AST_PRINT"): - bdv: black.DebugVisitor[Any] - black.out("Expected tree:", fg="green") + bdv: DebugVisitor[Any] + out("Expected tree:", fg="green") try: exp_node = black.lib2to3_parse(expected) - bdv = black.DebugVisitor() + bdv = DebugVisitor() list(bdv.visit(exp_node)) except Exception as ve: - black.err(str(ve)) - black.out("Actual tree:", fg="red") + err(str(ve)) + out("Actual tree:", fg="red") try: exp_node = black.lib2to3_parse(actual) - bdv = black.DebugVisitor() + bdv = DebugVisitor() list(bdv.visit(exp_node)) except Exception as ve: - black.err(str(ve)) + err(str(ve)) self.assertMultiLineEqual(expected, actual) From 036bea4aa0e2b9b3fe50d0d49addc811cce61fa4 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sat, 8 May 2021 05:34:25 -0400 Subject: [PATCH 60/81] Speed up tests even more (#2205) There's three optimizations in this commit: 1. Don't check if Black's output is stable or equivalant if no changes were made in the first place. It's not like passing the same code (for both source and actual) through black.assert_equivalent or black.assert_stable is useful. It's not a big deal for the smaller tests, but it eats a lot of time in tests/test_format.py since its test cases are big. This is also closer to how Black works IRL. 2. Use a smaller file for `test_root_logger_not_used_directly` since the logging it's checking happens during blib2to3's startup so the file doesn't really matter. 3. If we're checking a file is formatting (i.e. test_source_is_formatted) don't run Black over it again with `black.format_file_in_place`. `tests/test_format.py::TestSimpleFormat.check_file` is good enough. --- tests/test_black.py | 7 ++++--- tests/test_format.py | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 29a0731dd9d..347eaf621fe 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -175,8 +175,9 @@ def test_piping(self) -> None: ) self.assertEqual(result.exit_code, 0) self.assertFormatEqual(expected, result.output) - black.assert_equivalent(source, result.output) - black.assert_stable(source, result.output, DEFAULT_MODE) + if source != result.output: + black.assert_equivalent(source, result.output) + black.assert_stable(source, result.output, DEFAULT_MODE) def test_piping_diff(self) -> None: diff_header = re.compile( @@ -1904,7 +1905,7 @@ def fail(*args: Any, **kwargs: Any) -> None: critical=fail, log=fail, ): - ff(THIS_FILE) + ff(THIS_DIR / "util.py") def test_invalid_config_return_code(self) -> None: tmp_file = Path(black.dump_to_file()) diff --git a/tests/test_format.py b/tests/test_format.py index e59d5218d2e..5c78afe0ba6 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -7,7 +7,6 @@ from tests.util import ( BlackBaseTestCase, fs, - ff, DEFAULT_MODE, dump_to_stderr, read_data, @@ -135,11 +134,11 @@ def test_experimental_format(self, filename: str) -> None: def test_source_is_formatted(self, filename: str) -> None: path = THIS_DIR.parent / filename self.check_file(str(path), DEFAULT_MODE, data=False) - self.assertFalse(ff(path)) def check_file(self, filename: str, mode: black.Mode, *, data: bool = True) -> None: source, expected = read_data(filename, data=data) actual = fs(source, mode=mode) self.assertFormatEqual(expected, actual) - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, mode) + if source != actual: + black.assert_equivalent(source, actual) + black.assert_stable(source, actual, mode) From 62bfbd6a63dcac2f6f31eb014f69397c9eb967d2 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sat, 8 May 2021 15:17:38 -0400 Subject: [PATCH 61/81] Reorganize docs v2 (GH-2174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I know I know, this is the second reorganization of the docs. I'm not saying the first one was bad or anything... but.. actually wait nah, *it was bad*. Anyway, welcome to probably my biggest commit. The main thing with this reorganization was to introduce nesting to the documentation! Having all of the docs be part of the main TOC was becoming too much. There wasn't much room to expand either. Finally, the old setup required a documentation generation step which was just annoying. The goals of this reorganization was to: 1. Significantly restructure the docs to be discoverable and understandable 2. Add room for further docs (like guides or contributing docs) 3. Get rid of the doc generation step (it was slow and frustrating) 4. Unblock other improvements and also just make contributing to the docs easier Another important change with this is that we are no longer using GitHub as a documentation host. While GitHub does support Markdown based docs actually pretty well, the lack of any features outside of GitHub Flavoured Markdown is quite limiting. ReadTheDocs is just much better suited for documentation. You can use reST, MyST, CommonMark, and all of their great features like toctrees and admonitions. Related to this change, we're adopting MyST as our flavour of Markdown. MyST introduces neat syntax extensions to Markdown that pretty much gives us the best of both worlds. The ease of use and simplicity of MD and the flexibility and expressiveness of reST. Also recommonmark is deprecated now. This switch was possible now we don't use GH as a docs host. MyST docs have to be built to really be usable / pretty, so the MD docs are going to look pretty bad on GH, but that's fine now! Another thing that should be noted is that the README has been stripped of most content since it was confusing. Users would read the README and then think some feature or bug was fixed already and is available in a release when in reality, they weren't. They were reading effectively the latest docs without knowing. See also: https://github.com/psf/black/issues/1759 FYI: CommonMark is a rationalized version of Markdown syntax -- Commit history before merge: * Switch to MyST-Parser + doc config cleanup recommonmark is being deprecated in favour of MyST-Parser. This change is welcomed, especially since MyST-Parser has syntax extensions for the Commonmark standard. Effectively we get to use a language that's powerful and expressive like ReST, but get the simplicity of Markdown. The rest of this effort will be using some MyST features. This reorganization efforts aims to remove as much duplication as possible. The regeneration step once needed is gone, significantly simplifing our Sphinx documentation configuration. * Tell pipenv we replaced recommonmark for MyST-Parser Also update `docs/requirements.txt` * Delete all auto generated content * Switch prettier for mdformat (plus a few plugins) **FYI: THIS WAS EFFECTIVELY REVERTED, SEE THIRD TO LAST COMMIT** prettier doesn't support MyST's syntax extensions which are going to be used in this reorganization effort so we have to switch formatter. Unfortanately mdformat's style is different from prettier's so time to reformat the whole repo too. We're excluding .github/ISSUE_TEMPLATE because I have no idea whether its changes are safe, so let's play it safe. * Fix the heading levels in CHANGES.md + a link MyST-Parser / sphinx's linkcheck complains otherwise. * Move reference docs into a docs/contributing dir They're for contributors of Black anyway. Also added a note in the summary document warning about the lack of attention the reference has been dealing with. * Rewrite and setup the new landing page + main TOC - add some more detail about Black's beta status - add licensing info - add external links in the main TOC for GitHub, PyPI, and IRC - prepare main TOC for new structure * Break out AUTHORS into its own file Not only was the AUTHORS list quite long, this makes it easy to include it in the Sphinx docs with just a simple symlink. * Add license to docs via a simple include Yes the document is orphaned but it is linked to in the landing page (docs/index.rst). * Add "The Black Code Style" section This mostly was a restructuring commit, there has been a few updates but not many. The main goal was to split "current style" and "planned changes to the style that haven't happened yet" to avoid confusion. * Add "Getting Started" page This is basically a quick start + even more. This commit is certainly one of most creatively involved in this effort. * Add "Usage and Configuration" section This commit was as much restructuring as new content. Instead of being in one giant file, usage and configuration documentation can expand without bloating a single file. * Add "Integrations" section Just a restructuring commit ... * Add "Guides" section This is a promising area of documentation that could easily grow in the future, let's prepare for that! * Add "Contributing" section This is also another area that I expect to see significant growth in. Contributors to Black could definitely do with some more specific docs that clears up certain parts of our slightly confusing project (it's only confusing because we're getting big and old!). * Rewrite CONTRIBUTING.md to just point to RTD * Rewrite README.md to delegate most info to RTD * Address feedback + a lot of corrections and edits I know I said I wanted to do these after landing this but given there's going to be no time between this being merged and a release getting pushed, I want these changes to make it in. - drop the number flag for mdformat - to reduce diffs, see also: https://mdformat.readthedocs.io/en/stable/users/style.html#ordered-lists - the GH issue templates should be safe by mdformat, so get rid of the exclude - clarify our configuration position - i.e. stop claiming we don't have many options, instead say we want as little formatting knobs as possible - lots and lots of punctuation, spelling, and grammar corrections (thanks Jelle!) - use RTD as the source for the CHANGELOG too - visual style cleanups - add docs about our .gitignore behaviour - expand GHA Action docs - clarify we want the PR number in the CHANGELOG entry - claify Black's behaviour for with statements post Python 3.9 - italicize a bunch of "Black"s Thank you goes to Jelle, Taneli (hukkinj1 on GH), Felix (felix-hilden on GH), and Wouter (wbolster on GH) for the feedback! * Merge remote-tracking branch 'upstream/master' into reorganize-docs-v2 merge conflicts suck, although these ones weren't too bad. * Add changelog entry + fix merge conflict resolution error I consider this important enough to be worthy of a changelog entry :) * Merge branch 'master' into reorganize-docs-v2 Co-authored-by: Łukasz Langa * Actually let's continue using prettier Prettier works fine for all of the default MyST syntax so let's not rock the boat as much. Dropping the mdformat commit was merge-conflict filled so here's additional commit instead. * Address Cooper's, Taneli's, and Jelle's feedback Lots of wording improvements by Cooper. Taneli suggested to disable the enabled by default MyST syntax not supported by Prettier and I agreed. And Jelle found one more spelling error! * More minor fixes --- .github/ISSUE_TEMPLATE/bug_report.md | 14 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- AUTHORS.md | 184 ++++++ CHANGES.md | 94 +-- CONTRIBUTING.md | 114 +--- Pipfile | 2 +- Pipfile.lock | 131 +++- README.md | 621 ++---------------- docs/authors.md | 188 +----- docs/black_primer.md | 120 ---- docs/conf.py | 229 +------ docs/contributing/gauging_changes.md | 61 ++ docs/contributing/index.rst | 39 ++ .../reference/reference_classes.rst | 0 .../reference/reference_exceptions.rst | 0 .../reference/reference_functions.rst | 0 .../reference/reference_summary.rst | 5 + .../the_basics.md} | 35 +- docs/getting_started.md | 49 ++ docs/github_actions.md | 27 - docs/guides/index.rst | 14 + .../introducing_black_to_your_project.md | 50 ++ .../using_black_with_other_tools.md} | 36 +- docs/index.rst | 85 ++- docs/installation_and_usage.md | 195 ------ .../editors.md} | 52 +- docs/integrations/github_actions.md | 35 + docs/integrations/index.rst | 28 + docs/integrations/source_version_control.md | 14 + docs/license.rst | 6 + docs/pyproject_toml.md | 91 --- docs/requirements.txt | 5 +- docs/show_your_style.md | 19 - .../current_style.md} | 59 +- docs/the_black_code_style/future_style.md | 35 + docs/the_black_code_style/index.rst | 19 + .../black_as_a_server.md} | 6 +- .../file_collection_and_discovery.md} | 18 +- docs/usage_and_configuration/index.rst | 24 + docs/usage_and_configuration/the_basics.md | 227 +++++++ docs/version_control_integration.md | 28 - 41 files changed, 1200 insertions(+), 1761 deletions(-) create mode 100644 AUTHORS.md mode change 100644 => 120000 docs/authors.md delete mode 100644 docs/black_primer.md create mode 100644 docs/contributing/gauging_changes.md create mode 100644 docs/contributing/index.rst rename docs/{ => contributing}/reference/reference_classes.rst (100%) rename docs/{ => contributing}/reference/reference_exceptions.rst (100%) rename docs/{ => contributing}/reference/reference_functions.rst (100%) rename docs/{ => contributing}/reference/reference_summary.rst (51%) rename docs/{contributing_to_black.md => contributing/the_basics.md} (68%) create mode 100644 docs/getting_started.md delete mode 100644 docs/github_actions.md create mode 100644 docs/guides/index.rst create mode 100644 docs/guides/introducing_black_to_your_project.md rename docs/{compatible_configs.md => guides/using_black_with_other_tools.md} (92%) delete mode 100644 docs/installation_and_usage.md rename docs/{editor_integration.md => integrations/editors.md} (85%) create mode 100644 docs/integrations/github_actions.md create mode 100644 docs/integrations/index.rst create mode 100644 docs/integrations/source_version_control.md create mode 100644 docs/license.rst delete mode 100644 docs/pyproject_toml.md delete mode 100644 docs/show_your_style.md rename docs/{the_black_code_style.md => the_black_code_style/current_style.md} (91%) create mode 100644 docs/the_black_code_style/future_style.md create mode 100644 docs/the_black_code_style/index.rst rename docs/{blackd.md => usage_and_configuration/black_as_a_server.md} (98%) rename docs/{ignoring_unmodified_files.md => usage_and_configuration/file_collection_and_discovery.md} (61%) create mode 100644 docs/usage_and_configuration/index.rst create mode 100644 docs/usage_and_configuration/the_basics.md delete mode 100644 docs/version_control_integration.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f4a438622aa..3b59906594a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,21 +11,21 @@ assignees: "" **To Reproduce** Steps to reproduce the behavior: 1. Take this file '...' -2. Run _Black_ on it with these arguments '....' -3. See error +1. Run _Black_ on it with these arguments '....' +1. See error **Expected behavior** A clear and concise description of what you expected to happen. **Environment (please complete the following information):** -- Version: [e.g. master] -- OS and Python version: [e.g. Linux/Python 3.7.4rc1] +- Version: \[e.g. master\] +- OS and Python version: \[e.g. Linux/Python 3.7.4rc1\] **Does this bug also happen on master?** To answer this, you have two options: -1. Use the online formatter at https://black.now.sh/?version=master, which will use the - latest master branch. -2. Or run _Black_ on your machine: +1. Use the online formatter at , which will + use the latest master branch. +1. Or run _Black_ on your machine: - create a new virtualenv (make sure it's the same Python version); - clone this repository; - run `pip install -e .[d,python2]`; diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 00dd5dd8fe5..56c2f0d0185 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,7 +7,7 @@ assignees: "" --- **Is your feature request related to a problem? Please describe.** A clear and concise -description of what the problem is. Ex. I'm always frustrated when [...] +description of what the problem is. Ex. I'm always frustrated when \[...\] **Describe the solution you'd like** A clear and concise description of what you want to happen. diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 00000000000..cb79dec3de3 --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,184 @@ +# Authors + +Glued together by [Łukasz Langa](mailto:lukasz@langa.pl). + +Maintained with [Carol Willing](mailto:carolcode@willingconsulting.com), +[Carl Meyer](mailto:carl@oddbird.net), +[Jelle Zijlstra](mailto:jelle.zijlstra@gmail.com), +[Mika Naylor](mailto:mail@autophagy.io), +[Zsolt Dollenstein](mailto:zsol.zsol@gmail.com), +[Cooper Lees](mailto:me@cooperlees.com), and Richard Si. + +Multiple contributions by: + +- [Abdur-Rahmaan Janhangeer](mailto:arj.python@gmail.com) +- [Adam Johnson](mailto:me@adamj.eu) +- [Adam Williamson](mailto:adamw@happyassassin.net) +- [Alexander Huynh](mailto:github@grande.coffee) +- [Alex Vandiver](mailto:github@chmrr.net) +- [Allan Simon](mailto:allan.simon@supinfo.com) +- Anders-Petter Ljungquist +- [Andrew Thorp](mailto:andrew.thorp.dev@gmail.com) +- [Andrew Zhou](mailto:andrewfzhou@gmail.com) +- [Andrey](mailto:dyuuus@yandex.ru) +- [Andy Freeland](mailto:andy@andyfreeland.net) +- [Anthony Sottile](mailto:asottile@umich.edu) +- [Arjaan Buijk](mailto:arjaan.buijk@gmail.com) +- [Arnav Borbornah](mailto:arnavborborah11@gmail.com) +- [Artem Malyshev](mailto:proofit404@gmail.com) +- [Asger Hautop Drewsen](mailto:asgerdrewsen@gmail.com) +- [Augie Fackler](mailto:raf@durin42.com) +- [Aviskar KC](mailto:aviskarkc10@gmail.com) +- Batuhan Taşkaya +- [Benjamin Wohlwend](mailto:bw@piquadrat.ch) +- [Benjamin Woodruff](mailto:github@benjam.info) +- [Bharat Raghunathan](mailto:bharatraghunthan9767@gmail.com) +- [Brandt Bucher](mailto:brandtbucher@gmail.com) +- [Brett Cannon](mailto:brett@python.org) +- [Bryan Bugyi](mailto:bryan.bugyi@rutgers.edu) +- [Bryan Forbes](mailto:bryan@reigndropsfall.net) +- [Calum Lind](mailto:calumlind@gmail.com) +- [Charles](mailto:peacech@gmail.com) +- Charles Reid +- [Christian Clauss](mailto:cclauss@bluewin.ch) +- [Christian Heimes](mailto:christian@python.org) +- [Chuck Wooters](mailto:chuck.wooters@microsoft.com) +- [Chris Rose](mailto:offline@offby1.net) +- Codey Oxley +- [Cong](mailto:congusbongus@gmail.com) +- [Cooper Ry Lees](mailto:me@cooperlees.com) +- [Dan Davison](mailto:dandavison7@gmail.com) +- [Daniel Hahler](mailto:github@thequod.de) +- [Daniel M. Capella](mailto:polycitizen@gmail.com) +- Daniele Esposti +- [David Hotham](mailto:david.hotham@metaswitch.com) +- [David Lukes](mailto:dafydd.lukes@gmail.com) +- [David Szotten](mailto:davidszotten@gmail.com) +- [Denis Laxalde](mailto:denis@laxalde.org) +- [Douglas Thor](mailto:dthor@transphormusa.com) +- dylanjblack +- [Eli Treuherz](mailto:eli@treuherz.com) +- [Emil Hessman](mailto:emil@hessman.se) +- [Felix Kohlgrüber](mailto:felix.kohlgrueber@gmail.com) +- [Florent Thiery](mailto:fthiery@gmail.com) +- Francisco +- [Giacomo Tagliabue](mailto:giacomo.tag@gmail.com) +- [Greg Gandenberger](mailto:ggandenberger@shoprunner.com) +- [Gregory P. Smith](mailto:greg@krypto.org) +- Gustavo Camargo +- hauntsaninja +- [Hadi Alqattan](mailto:alqattanhadizaki@gmail.com) +- [Heaford](mailto:dan@heaford.com) +- [Hugo Barrera](mailto::hugo@barrera.io) +- Hugo van Kemenade +- [Hynek Schlawack](mailto:hs@ox.cx) +- [Ivan Katanić](mailto:ivan.katanic@gmail.com) +- [Jakub Kadlubiec](mailto:jakub.kadlubiec@skyscanner.net) +- [Jakub Warczarek](mailto:jakub.warczarek@gmail.com) +- [Jan Hnátek](mailto:jan.hnatek@gmail.com) +- [Jason Fried](mailto:me@jasonfried.info) +- [Jason Friedland](mailto:jason@friedland.id.au) +- [jgirardet](mailto:ijkl@netc.fr) +- Jim Brännlund +- [Jimmy Jia](mailto:tesrin@gmail.com) +- [Joe Antonakakis](mailto:jma353@cornell.edu) +- [Jon Dufresne](mailto:jon.dufresne@gmail.com) +- [Jonas Obrist](mailto:ojiidotch@gmail.com) +- [Jonty Wareing](mailto:jonty@jonty.co.uk) +- [Jose Nazario](mailto:jose.monkey.org@gmail.com) +- [Joseph Larson](mailto:larson.joseph@gmail.com) +- [Josh Bode](mailto:joshbode@fastmail.com) +- [Josh Holland](mailto:anowlcalledjosh@gmail.com) +- [Joshua Cannon](mailto:joshdcannon@gmail.com) +- [José Padilla](mailto:jpadilla@webapplicate.com) +- [Juan Luis Cano Rodríguez](mailto:hello@juanlu.space) +- [kaiix](mailto:kvn.hou@gmail.com) +- [Katie McLaughlin](mailto:katie@glasnt.com) +- Katrin Leinweber +- [Keith Smiley](mailto:keithbsmiley@gmail.com) +- [Kenyon Ralph](mailto:kenyon@kenyonralph.com) +- [Kevin Kirsche](mailto:Kev.Kirsche+GitHub@gmail.com) +- [Kyle Hausmann](mailto:kyle.hausmann@gmail.com) +- [Kyle Sunden](mailto:sunden@wisc.edu) +- Lawrence Chan +- [Linus Groh](mailto:mail@linusgroh.de) +- [Loren Carvalho](mailto:comradeloren@gmail.com) +- [Luka Sterbic](mailto:luka.sterbic@gmail.com) +- [LukasDrude](mailto:mail@lukas-drude.de) +- Mahmoud Hossam +- Mariatta +- [Matt VanEseltine](mailto:vaneseltine@gmail.com) +- [Matthew Clapp](mailto:itsayellow+dev@gmail.com) +- [Matthew Walster](mailto:matthew@walster.org) +- Max Smolens +- [Michael Aquilina](mailto:michaelaquilina@gmail.com) +- [Michael Flaxman](mailto:michael.flaxman@gmail.com) +- [Michael J. Sullivan](mailto:sully@msully.net) +- [Michael McClimon](mailto:michael@mcclimon.org) +- [Miguel Gaiowski](mailto:miggaiowski@gmail.com) +- [Mike](mailto:roshi@fedoraproject.org) +- [mikehoyio](mailto:mikehoy@gmail.com) +- [Min ho Kim](mailto:minho42@gmail.com) +- [Miroslav Shubernetskiy](mailto:miroslav@miki725.com) +- MomIsBestFriend +- [Nathan Goldbaum](mailto:ngoldbau@illinois.edu) +- [Nathan Hunt](mailto:neighthan.hunt@gmail.com) +- [Neraste](mailto:neraste.herr10@gmail.com) +- [Nikolaus Waxweiler](mailto:madigens@gmail.com) +- [Ofek Lev](mailto:ofekmeister@gmail.com) +- [Osaetin Daniel](mailto:osaetindaniel@gmail.com) +- [otstrel](mailto:otstrel@gmail.com) +- [Pablo Galindo](mailto:Pablogsal@gmail.com) +- [Paul Ganssle](mailto:p.ganssle@gmail.com) +- [Paul Meinhardt](mailto:mnhrdt@gmail.com) +- [Peter Bengtsson](mailto:mail@peterbe.com) +- [Peter Grayson](mailto:pete@jpgrayson.net) +- [Peter Stensmyr](mailto:peter.stensmyr@gmail.com) +- pmacosta +- [Quentin Pradet](mailto:quentin@pradet.me) +- [Ralf Schmitt](mailto:ralf@systemexit.de) +- [Ramón Valles](mailto:mroutis@protonmail.com) +- [Richard Fearn](mailto:richardfearn@gmail.com) +- [Rishikesh Jha](mailto:rishijha424@gmail.com) +- [Rupert Bedford](mailto:rupert@rupertb.com) +- Russell Davis +- [Rémi Verschelde](mailto:rverschelde@gmail.com) +- [Sami Salonen](mailto:sakki@iki.fi) +- [Samuel Cormier-Iijima](mailto:samuel@cormier-iijima.com) +- [Sanket Dasgupta](mailto:sanketdasgupta@gmail.com) +- Sergi +- [Scott Stevenson](mailto:scott@stevenson.io) +- Shantanu +- [shaoran](mailto:shaoran@sakuranohana.org) +- [Shinya Fujino](mailto:shf0811@gmail.com) +- springstan +- [Stavros Korokithakis](mailto:hi@stavros.io) +- [Stephen Rosen](mailto:sirosen@globus.org) +- [Steven M. Vascellaro](mailto:S.Vascellaro@gmail.com) +- [Sunil Kapil](mailto:snlkapil@gmail.com) +- [Sébastien Eustace](mailto:sebastien.eustace@gmail.com) +- [Tal Amuyal](mailto:TalAmuyal@gmail.com) +- [Terrance](mailto:git@terrance.allofti.me) +- [Thom Lu](mailto:thomas.c.lu@gmail.com) +- [Thomas Grainger](mailto:tagrain@gmail.com) +- [Tim Gates](mailto:tim.gates@iress.com) +- [Tim Swast](mailto:swast@google.com) +- [Timo](mailto:timo_tk@hotmail.com) +- Toby Fleming +- [Tom Christie](mailto:tom@tomchristie.com) +- [Tony Narlock](mailto:tony@git-pull.com) +- [Tsuyoshi Hombashi](mailto:tsuyoshi.hombashi@gmail.com) +- [Tushar Chandra](mailto:tusharchandra2018@u.northwestern.edu) +- [Tzu-ping Chung](mailto:uranusjr@gmail.com) +- [Utsav Shah](mailto:ukshah2@illinois.edu) +- utsav-dbx +- vezeli +- [Ville Skyttä](mailto:ville.skytta@iki.fi) +- [Vishwas B Sharma](mailto:sharma.vishwas88@gmail.com) +- [Vlad Emelianov](mailto:volshebnyi@gmail.com) +- [williamfzc](mailto:178894043@qq.com) +- [wouter bolsterlee](mailto:wouter@bolsterl.ee) +- Yazdan +- [Yngve Høiseth](mailto:yngve@hoiseth.net) +- [Yurii Karabas](mailto:1998uriyyo@gmail.com) +- [Zac Hatfield-Dodds](mailto:zac@zhd.dev) diff --git a/CHANGES.md b/CHANGES.md index 8cbdeca44f7..9b155924b7d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,26 +1,32 @@ -## Change Log +# Change Log -### Unreleased +## Unreleased -#### _Black_ +### _Black_ - Refactor `src/black/__init__.py` into many files (#2206) -### 21.5b0 +### Documentation -#### _Black_ +- Sigificantly reorganized the documentation to make much more sense. Check them out by + heading over to [the stable docs on RTD](https://black.readthedocs.io/en/stable/). + (#2174) + +## 21.5b0 + +### _Black_ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) - Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are being called but with no arguments (#2182) -#### _Black-Primer_ +### _Black-Primer_ - Add `--no-diff` to black-primer to suppress formatting changes (#2187) -### 21.4b2 +## 21.4b2 -#### _Black_ +### _Black_ - Fix crash if the user configuration directory is inaccessible. (#2158) @@ -31,15 +37,15 @@ - Allow `.gitignore` rules to be overridden by specifying `exclude` in `pyproject.toml` or on the command line. (#2170) -#### _Packaging_ +### _Packaging_ - Install `primer.json` (used by `black-primer` by default) with black. (#2154) -### 21.4b1 +## 21.4b1 -#### _Black_ +### _Black_ -- Fix crash on docstrings ending with "\ ". (#2142) +- Fix crash on docstrings ending with "\\ ". (#2142) - Fix crash when atypical whitespace is cleaned out of dostrings (#2120) @@ -50,13 +56,13 @@ - Don't remove necessary parentheses from assignment expression containing assert / return statements. (#2143) -#### _Packaging_ +### _Packaging_ - Bump pathspec to >= 0.8.1 to solve invalid .gitignore exclusion handling -### 21.4b0 +## 21.4b0 -#### _Black_ +### _Black_ - Fixed a rare but annoying formatting instability created by the combination of optional trailing commas inserted by `Black` and optional parentheses looking at @@ -112,21 +118,21 @@ - Fixed "Black produced code that is not equivalent to the source" when formatting Python 2 docstrings (#2037) -#### _Packaging_ +### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub Releases (#1743) -### 20.8b1 +## 20.8b1 -#### _Packaging_ +### _Packaging_ - explicitly depend on Click 7.1.2 or newer as `Black` no longer works with versions older than 7.0 -### 20.8b0 +## 20.8b0 -#### _Black_ +### _Black_ - re-implemented support for explicit trailing commas: now it works consistently within any bracket pair, including nested structures (#1288 and duplicates) @@ -170,11 +176,11 @@ this is an undocumented and unsupported feature, you lose Internet points for depending on it (#1609) -#### Vim plugin +### Vim plugin - prefer virtualenv packages over global packages (#1383) -### 19.10b0 +## 19.10b0 - added support for PEP 572 assignment expressions (#711) @@ -233,7 +239,7 @@ - `blackd` can now output the diff of formats on source code when the `X-Diff` header is provided (#969) -### 19.3b0 +## 19.3b0 - new option `--target-version` to control which Python versions _Black_-formatted code should target (#618) @@ -258,7 +264,7 @@ - `blackd` now supports CORS (#622) -### 18.9b0 +## 18.9b0 - numeric literals are now formatted by _Black_ (#452, #461, #464, #469): @@ -273,7 +279,9 @@ - hexadecimal digits are always uppercased (e.g. `0xBADC0DE`) -- added `blackd`, see [its documentation](#blackd) for more info (#349) +- added `blackd`, see + [its documentation](https://github.com/psf/black/blob/18.9b0/README.md#blackd) for + more info (#349) - adjacent string literals are now correctly split into multiple lines (#463) @@ -300,11 +308,11 @@ - note: the Vim plugin stopped registering `,=` as a default chord as it turned out to be a bad idea (#415) -### 18.6b4 +## 18.6b4 - hotfix: don't freeze when multiple comments directly precede `# fmt: off` (#371) -### 18.6b3 +## 18.6b3 - typing stub files (`.pyi`) now have blank lines added after constants (#340) @@ -332,7 +340,7 @@ - fixed a crash due to symbolic links pointing outside of the project directory (#338) -### 18.6b2 +## 18.6b2 - added `--config` (#65) @@ -346,13 +354,13 @@ - fixed unnecessary slowdown in comment placement calculation on lines without comments -### 18.6b1 +## 18.6b1 - hotfix: don't output human-facing information on stdout (#299) - hotfix: don't output cake emoji on non-zero return code (#300) -### 18.6b0 +## 18.6b0 - added `--include` and `--exclude` (#270) @@ -370,7 +378,7 @@ - _Black_ now preserves line endings when formatting a file in place (#258) -### 18.5b1 +## 18.5b1 - added `--pyi` (#249) @@ -399,7 +407,7 @@ - fixed extra empty line between a function signature and an inner function or inner class (#196) -### 18.5b0 +## 18.5b0 - call chains are now formatted according to the [fluent interfaces](https://en.wikipedia.org/wiki/Fluent_interface) style (#67) @@ -454,11 +462,11 @@ - fixed crash when dead symlinks where encountered -### 18.4a4 +## 18.4a4 - don't populate the cache on `--check` (#175) -### 18.4a3 +## 18.4a3 - added a "cache"; files already reformatted that haven't changed on disk won't be reformatted again (#109) @@ -486,7 +494,7 @@ - fixed missing splits of ternary expressions (#141) -### 18.4a2 +## 18.4a2 - fixed parsing of unaligned standalone comments (#99, #112) @@ -497,7 +505,7 @@ - fixed unstable formatting when encountering unnecessarily escaped quotes in a string (#120) -### 18.4a1 +## 18.4a1 - added `--quiet` (#78) @@ -509,7 +517,7 @@ - fixed removing backslash escapes from raw strings (#100, #105) -### 18.4a0 +## 18.4a0 - added `--diff` (#87) @@ -534,7 +542,7 @@ - only allow up to two empty lines on module level and only single empty lines within functions (#74) -### 18.3a4 +## 18.3a4 - `# fmt: off` and `# fmt: on` are implemented (#5) @@ -556,7 +564,7 @@ [Sphinx auto-attribute comments](http://www.sphinx-doc.org/en/stable/ext/autodoc.html#directive-autoattribute) (#68) -### 18.3a3 +## 18.3a3 - don't remove single empty lines outside of bracketed expressions (#19) @@ -566,7 +574,7 @@ - even better handling of numpy-style array indexing (#33, again) -### 18.3a2 +## 18.3a2 - changed positioning of binary operators to occur at beginning of lines instead of at the end, following @@ -590,7 +598,7 @@ - fixed spurious space after star-based unary expressions (#31) -### 18.3a1 +## 18.3a1 - added `--check` @@ -609,10 +617,10 @@ - fixed spurious space after unary operators when the operand was a complex expression (#15) -### 18.3a0 +## 18.3a0 - first published version, Happy 🍰 Day 2018! - alpha quality -- date-versioned (see: https://calver.org/) +- date-versioned (see: ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2a4ee20c99..10f60422f04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,114 +3,8 @@ Welcome! Happy to see you willing to make the project better. Have you read the entire [user documentation](https://black.readthedocs.io/en/latest/) yet? -## Bird's eye view +Our [contributing documentation](https://black.readthedocs.org/en/latest/contributing/) +contains details on all you need to know about contributing to _Black_, the basics to +the internals of _Black_. -In terms of inspiration, _Black_ is about as configurable as _gofmt_. This is -deliberate. - -Bug reports and fixes are always welcome! Please follow the -[issue template on GitHub](https://github.com/psf/black/issues/new) for best results. - -Before you suggest a new feature or configuration knob, ask yourself why you want it. If -it enables better integration with some workflow, fixes an inconsistency, speeds things -up, and so on - go for it! On the other hand, if your answer is "because I don't like a -particular formatting" then you're not ready to embrace _Black_ yet. Such changes are -unlikely to get accepted. You can still try but prepare to be disappointed. - -## Technicalities - -Development on the latest version of Python is preferred. As of this writing it's 3.9. -You can use any operating system. I am using macOS myself and CentOS at work. - -Install all development dependencies using: - -```console -$ pipenv install --dev -$ pipenv shell -$ pre-commit install -``` - -If you haven't used `pipenv` before but are comfortable with virtualenvs, just run -`pip install pipenv` in the virtualenv you're already using and invoke the command above -from the cloned _Black_ repo. It will do the correct thing. - -Non pipenv install works too: - -```console -$ pip install -r test_requirements -$ pip install -e .[d] -``` - -Before submitting pull requests, run lints and tests with the following commands from -the root of the black repo: - -```console -# Linting -$ pre-commit run -a - -# Unit tests -$ tox -e py - -# Optional Fuzz testing -$ tox -e fuzz - -# Optional CI run to test your changes on many popular python projects -$ black-primer [-k -w /tmp/black_test_repos] -``` - -### News / Changelog Requirement - -`Black` has CI that will check for an entry corresponding to your PR in `CHANGES.md`. If -you feel this PR not require a changelog entry please state that in a comment and a -maintainer can add a `skip news` label to make the CI pass. Otherwise, please ensure you -have a line in the following format: - -```md -- `Black` is now more awesome (#X) -``` - -To workout X, please use -[Next PR Number](https://ichard26.github.io/next-pr-number/?owner=psf&name=black). This -is not perfect but saves a lot of release overhead as now the releaser does not need to -go back and workout what to add to the `CHANGES.md` for each release. - -### Style Changes - -If a change would affect the advertised code style, please modify the documentation (The -_Black_ code style) to reflect that change. Patches that fix unintended bugs in -formatting don't need to be mentioned separately though. - -### Docs Testing - -If you make changes to docs, you can test they still build locally too. - -```console -$ pip install -r docs/requirements.txt -$ pip install [-e] .[d] -$ sphinx-build -a -b html -W docs/ docs/_build/ -``` - -## black-primer - -`black-primer` is used by CI to pull down well-known _Black_ formatted projects and see -if we get source code changes. It will error on formatting changes or errors. Please run -before pushing your PR to see if you get the actions you would expect from _Black_ with -your PR. You may need to change -[primer.json](https://github.com/psf/black/blob/master/src/black_primer/primer.json) -configuration for it to pass. - -For more `black-primer` information visit the -[documentation](https://github.com/psf/black/blob/master/docs/black_primer.md). - -## Hygiene - -If you're fixing a bug, add a test. Run it first to confirm it fails, then fix the bug, -run it again to confirm it's really fixed. - -If adding a new feature, add a test. In fact, always add a test. But wait, before adding -any large feature, first open an issue for us to discuss the idea first. - -## Finally - -Thanks again for your interest in improving the project! You're taking action when most -people decide to sit and watch. +We look forward to your contributions! diff --git a/Pipfile b/Pipfile index 68562f5e53f..d1842ff749b 100644 --- a/Pipfile +++ b/Pipfile @@ -12,7 +12,7 @@ flake8-bugbear = "*" mypy = ">=0.812" pre-commit = "*" readme_renderer = "*" -recommonmark = "*" +MyST-Parser = ">=0.13.7" setuptools = ">=39.2.0" setuptools-scm = "*" twine = ">=1.11.0" diff --git a/Pipfile.lock b/Pipfile.lock index 8a16a75026a..f973c8423e8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1b075e0e344dad54944ed4474351c3f311accb67cbea2e0c381da3700b71f415" + "sha256": "62bc4bdb0117234d1f374b2dc0685369f6df7c7192d16409cc9c42a429770166" }, "pipfile-spec": 6, "requires": {}, @@ -428,11 +428,11 @@ }, "babel": { "hashes": [ - "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5", - "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05" + "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9", + "sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.9.0" + "version": "==2.9.1" }, "black": { "editable": true, @@ -456,6 +456,48 @@ ], "version": "==2020.12.5" }, + "cffi": { + "hashes": [ + "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813", + "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06", + "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea", + "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee", + "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396", + "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73", + "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315", + "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1", + "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49", + "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892", + "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482", + "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058", + "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5", + "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53", + "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045", + "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3", + "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5", + "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e", + "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c", + "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369", + "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827", + "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053", + "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa", + "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4", + "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322", + "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132", + "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62", + "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa", + "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0", + "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396", + "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e", + "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991", + "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6", + "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1", + "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406", + "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d", + "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c" + ], + "version": "==1.14.5" + }, "cfgv": { "hashes": [ "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", @@ -488,13 +530,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.4.4" }, - "commonmark": { - "hashes": [ - "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", - "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" - ], - "version": "==0.9.1" - }, "coverage": { "hashes": [ "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", @@ -553,6 +588,24 @@ "index": "pypi", "version": "==5.5" }, + "cryptography": { + "hashes": [ + "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", + "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", + "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", + "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", + "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", + "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", + "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", + "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", + "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", + "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", + "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", + "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" + ], + "markers": "python_version >= '3.6'", + "version": "==3.4.7" + }, "distlib": { "hashes": [ "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", @@ -624,6 +677,14 @@ "markers": "python_version >= '3.6'", "version": "==4.0.1" }, + "jeepney": { + "hashes": [ + "sha256:7d59b6622675ca9e993a6bd38de845051d315f8b0c72cca3aef733a20b648657", + "sha256:aec56c0eb1691a841795111e184e13cad504f7703b9a64f63020816afa79a8ae" + ], + "markers": "sys_platform == 'linux'", + "version": "==0.6.0" + }, "jinja2": { "hashes": [ "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", @@ -640,6 +701,14 @@ "markers": "python_version >= '3.6'", "version": "==23.0.1" }, + "markdown-it-py": { + "hashes": [ + "sha256:30b3e9f8198dc82a5df0dcb73fd31d56cd9a43bf8a747feb10b2ba74f962bcb1", + "sha256:c3b9f995be0792cbbc8ab2f53d74072eb7ff8a8b622be8d61d38ab879709eca3" + ], + "markers": "python_version ~= '3.6'", + "version": "==0.6.2" + }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -705,6 +774,14 @@ ], "version": "==0.6.1" }, + "mdit-py-plugins": { + "hashes": [ + "sha256:1e467ca2ea056e8065cbd5d6c61e5052bb50826bde84c40f6a5ed77e82125710", + "sha256:77fd75dad81109ee91f30eb49146196f79afbbae041f298ae4886c8c2b5e23d7" + ], + "markers": "python_version ~= '3.6'", + "version": "==0.2.6" + }, "multidict": { "hashes": [ "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a", @@ -784,6 +861,14 @@ "index": "pypi", "version": "==0.4.3" }, + "myst-parser": { + "hashes": [ + "sha256:260355b4da8e8865fe080b0638d7f1ab1791dc4bed02a7a48630b6bad4249219", + "sha256:e4bc99e43e19f70d22e528de8e7cce59f7e8e7c4c34dcba203de92de7a7c7c85" + ], + "index": "pypi", + "version": "==0.13.7" + }, "nodeenv": { "hashes": [ "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b", @@ -830,6 +915,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7.0" }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, "pyflakes": { "hashes": [ "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", @@ -904,14 +997,6 @@ "index": "pypi", "version": "==29.0" }, - "recommonmark": { - "hashes": [ - "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f", - "sha256:bdb4db649f2222dcd8d2d844f0006b958d627f732415d399791ee436a3686d67" - ], - "index": "pypi", - "version": "==0.7.1" - }, "regex": { "hashes": [ "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", @@ -981,6 +1066,14 @@ ], "version": "==1.4.0" }, + "secretstorage": { + "hashes": [ + "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", + "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" + ], + "markers": "sys_platform == 'linux'", + "version": "==3.3.1" + }, "setuptools-scm": { "hashes": [ "sha256:c3bd5f701c8def44a5c0bfe8d407bef3f80342217ef3492b951f3777bd2d915c", diff --git a/README.md b/README.md index 6443569d0d0..ceccbb79962 100644 --- a/README.md +++ b/README.md @@ -26,21 +26,12 @@ becomes transparent after a while and you can focus on the content instead. _Black_ makes code review faster by producing the smallest diffs possible. -Try it out now using the [Black Playground](https://black.now.sh). Watch the +Try it out now using the [Black Playground](https://black.vercel.app). Watch the [PyCon 2019 talk](https://youtu.be/esZLCuWs_2Y) to learn more. --- -_Contents:_ **[Installation and usage](#installation-and-usage)** | -**[Code style](#the-black-code-style)** | **[Pragmatism](#pragmatism)** | -**[pyproject.toml](#pyprojecttoml)** | **[Editor integration](#editor-integration)** | -**[blackd](#blackd)** | **[black-primer](#black-primer)** | -**[Version control integration](#version-control-integration)** | -**[GitHub Actions](#github-actions)** | -**[Ignoring unmodified files](#ignoring-unmodified-files)** | **[Used by](#used-by)** | -**[Testimonials](#testimonials)** | **[Show your style](#show-your-style)** | -**[Contributing](#contributing-to-black)** | **[Change log](#change-log)** | -**[Authors](#authors)** +**[Read the documentation on ReadTheDocs!](https://black.readthedocs.io/en/stable)** --- @@ -52,8 +43,6 @@ _Black_ can be installed by running `pip install black`. It requires Python 3.6. run. If you want to format Python 2 code as well, install with `pip install black[python2]`. -#### Install from GitHub - If you can't wait for the latest _hotness_ and want to install from GitHub, use: `pip install git+git://github.com/psf/black` @@ -72,392 +61,72 @@ You can run _Black_ as a package if running it as a script doesn't work: python -m black {source_file_or_directory} ``` -### Command line options - -_Black_ doesn't provide many options. You can list them by running `black --help`: - -```text -Usage: black [OPTIONS] [SRC]... - - The uncompromising code formatter. - -Options: - -c, --code TEXT Format the code passed in as a string. - -l, --line-length INTEGER How many characters per line to allow. - [default: 88] - - -t, --target-version [py27|py33|py34|py35|py36|py37|py38|py39] - Python versions that should be supported by - Black's output. [default: per-file auto- - detection] - - --pyi Format all input files like typing stubs - regardless of file extension (useful when - piping source on standard input). - - -S, --skip-string-normalization - Don't normalize string quotes or prefixes. - -C, --skip-magic-trailing-comma - Don't use trailing commas as a reason to - split lines. - - --check Don't write the files back, just return the - status. Return code 0 means nothing would - change. Return code 1 means some files - would be reformatted. Return code 123 means - there was an internal error. - - --diff Don't write the files back, just output a - diff for each file on stdout. - - --color / --no-color Show colored diff. Only applies when - `--diff` is given. - - --fast / --safe If --fast given, skip temporary sanity - checks. [default: --safe] - - --include TEXT A regular expression that matches files and - directories that should be included on - recursive searches. An empty value means - all files are included regardless of the - name. Use forward slashes for directories - on all platforms (Windows, too). Exclusions - are calculated first, inclusions later. - [default: \.pyi?$] - - --exclude TEXT A regular expression that matches files and - directories that should be excluded on - recursive searches. An empty value means no - paths are excluded. Use forward slashes for - directories on all platforms (Windows, too). - Exclusions are calculated first, inclusions - later. [default: /(\.direnv|\.eggs|\.git|\. - hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.sv - n|_build|buck-out|build|dist)/] - - --extend-exclude TEXT Like --exclude, but adds additional files - and directories on top of the excluded - ones (useful if you simply want to add to - the default). - - --force-exclude TEXT Like --exclude, but files and directories - matching this regex will be excluded even - when they are passed explicitly as - arguments. - - - --stdin-filename TEXT The name of the file when passing it through - stdin. Useful to make sure Black will - respect --force-exclude option on some - editors that rely on using stdin. - - -q, --quiet Don't emit non-error messages to stderr. - Errors are still emitted; silence those with - 2>/dev/null. - - -v, --verbose Also emit messages to stderr about files - that were not changed or were ignored due to - exclusion patterns. - - --version Show the version and exit. - --config FILE Read configuration from FILE path. - -h, --help Show this message and exit. -``` - -_Black_ is a well-behaved Unix-style command-line tool: - -- it does nothing if no sources are passed to it; -- it will read from standard input and write to standard output if `-` is used as the - filename; -- it only outputs messages to users on standard error; -- exits with code 0 unless an internal error occurred (or `--check` was used). - -### Using _Black_ with other tools - -While _Black_ enforces formatting that conforms to PEP 8, other tools may raise warnings -about _Black_'s changes or will overwrite _Black_'s changes. A good example of this is -[isort](https://pypi.org/p/isort). Since _Black_ is barely configurable, these tools -should be configured to neither warn about nor overwrite _Black_'s changes. - -Actual details on _Black_ compatible configurations for various tools can be found in -[compatible_configs](https://github.com/psf/black/blob/master/docs/compatible_configs.md#black-compatible-configurations). - -### Migrating your code style without ruining git blame - -A long-standing argument against moving to automated code formatters like _Black_ is -that the migration will clutter up the output of `git blame`. This was a valid argument, -but since Git version 2.23, Git natively supports -[ignoring revisions in blame](https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revltrevgt) -with the `--ignore-rev` option. You can also pass a file listing the revisions to ignore -using the `--ignore-revs-file` option. The changes made by the revision will be ignored -when assigning blame. Lines modified by an ignored revision will be blamed on the -previous revision that modified those lines. - -So when migrating your project's code style to _Black_, reformat everything and commit -the changes (preferably in one massive commit). Then put the full 40 characters commit -identifier(s) into a file. - -``` -# Migrate code style to Black -5b4ab991dede475d393e9d69ec388fd6bd949699 -``` - -Afterwards, you can pass that file to `git blame` and see clean and meaningful blame -information. - -```console -$ git blame important.py --ignore-revs-file .git-blame-ignore-revs -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 1) def very_important_function(text, file): -abdfd8b0 (Alice Doe 2019-09-23 11:39:32 -0400 2) text = text.lstrip() -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 3) with open(file, "r+") as f: -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 4) f.write(formatted) -``` - -You can even configure `git` to automatically ignore revisions listed in a file on every -call to `git blame`. - -```console -$ git config blame.ignoreRevsFile .git-blame-ignore-revs -``` +Further information can be found in our docs: -**The one caveat is that GitHub and GitLab do not yet support ignoring revisions using -their native UI of blame.** So blame information will be cluttered with a reformatting -commit on those platforms. (If you'd like this feature, there's an open issue for -[GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423) and please let GitHub -know!) +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) ### NOTE: This is a beta product _Black_ is already [successfully used](https://github.com/psf/black#used-by) by many -projects, small and big. It also sports a decent test suite. However, it is still very -new. Things will probably be wonky for a while. This is made explicit by the "Beta" -trove classifier, as well as by the "b" in the version number. What this means for you -is that **until the formatter becomes stable, you should expect some formatting to -change in the future**. That being said, no drastic stylistic changes are planned, -mostly responses to bug reports. +projects, small and big. Black has a comprehensive test suite, with efficient parallel +tests, and our own auto formatting and parallel Continuous Integration runner. However, +_Black_ is still beta. Things will probably be wonky for a while. This is made explicit +by the "Beta" trove classifier, as well as by the "b" in the version number. What this +means for you is that **until the formatter becomes stable, you should expect some +formatting to change in the future**. That being said, no drastic stylistic changes are +planned, mostly responses to bug reports. Also, as a safety measure which slows down processing, _Black_ will check that the reformatted code still produces a valid AST that is effectively equivalent to the original (see the -[Pragmatism](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) +[Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#ast-before-and-after-formatting) section for details). If you're feeling confident, use `--fast`. ## The _Black_ code style _Black_ is a PEP 8 compliant opinionated formatter. _Black_ reformats entire files in -place. It is not configurable. It doesn't take previous formatting into account. Your -main option of configuring _Black_ is that it doesn't reformat blocks that start with -`# fmt: off` and end with `# fmt: on`, or lines that ends with `# fmt: skip`. Pay -attention that `# fmt: on/off` have to be on the same level of indentation. To learn -more about _Black_'s opinions, to go -[the_black_code_style](https://github.com/psf/black/blob/master/docs/the_black_code_style.md). +place. Style configuration options are deliberately limited and rarely added. It doesn't +take previous formatting into account (see [Pragmatism](#pragmatism) for exceptions). + +Our documentation covers the current _Black_ code style, but planned changes to it are +also documented. They're both worth taking a look: + +- [The _Black_ Code Style: Current style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) +- [The _Black_ Code Style: Future style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html) Please refer to this document before submitting an issue. What seems like a bug might be intended behaviour. -## Pragmatism +### Pragmatism Early versions of _Black_ used to be absolutist in some respects. They took after its initial author. This was fine at the time as it made the implementation simpler and there were not many users anyway. Not many edge cases were reported. As a mature tool, -_Black_ does make some exceptions to rules it otherwise holds. This -[section](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) -of `the_black_code_style` describes what those exceptions are and why this is the case. +_Black_ does make some exceptions to rules it otherwise holds. + +- [The _Black_ code style: Pragmatism](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#pragmatism) Please refer to this document before submitting an issue just like with the document above. What seems like a bug might be intended behaviour. -## pyproject.toml +## Configuration _Black_ is able to read project-specific default values for its command line options from a `pyproject.toml` file. This is especially useful for specifying custom -`--include` and `--exclude`/`--extend-exclude` patterns for your project. - -**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is -"No". _Black_ is all about sensible defaults. - -### What on Earth is a `pyproject.toml` file? - -[PEP 518](https://www.python.org/dev/peps/pep-0518/) defines `pyproject.toml` as a -configuration file to store build system requirements for Python projects. With the help -of tools like [Poetry](https://python-poetry.org/) or -[Flit](https://flit.readthedocs.io/en/latest/) it can fully replace the need for -`setup.py` and `setup.cfg` files. - -### Where _Black_ looks for the file - -By default _Black_ looks for `pyproject.toml` starting from the common base directory of -all files and directories passed on the command line. If it's not there, it looks in -parent directories. It stops looking when it finds the file, or a `.git` directory, or a -`.hg` directory, or the root of the file system, whichever comes first. - -If you're formatting standard input, _Black_ will look for configuration starting from -the current working directory. - -You can use a "global" configuration, stored in a specific location in your home -directory. This will be used as a fallback configuration, that is, it will be used if -and only if _Black_ doesn't find any configuration as mentioned above. Depending on your -operating system, this configuration file should be stored as: - -- Windows: `~\.black` -- Unix-like (Linux, MacOS, etc.): `$XDG_CONFIG_HOME/black` (`~/.config/black` if the - `XDG_CONFIG_HOME` environment variable is not set) - -Note that these are paths to the TOML file itself (meaning that they shouldn't be named -as `pyproject.toml`), not directories where you store the configuration. Here, `~` -refers to the path to your home directory. On Windows, this will be something like -`C:\\Users\UserName`. - -You can also explicitly specify the path to a particular file that you want with -`--config`. In this situation _Black_ will not look for any other file. - -If you're running with `--verbose`, you will see a blue message if a file was found and -used. - -Please note `blackd` will not use `pyproject.toml` configuration. - -### Configuration format - -As the file extension suggests, `pyproject.toml` is a -[TOML](https://github.com/toml-lang/toml) file. It contains separate sections for -different tools. _Black_ is using the `[tool.black]` section. The option keys are the -same as long names of options on the command line. - -Note that you have to use single-quoted strings in TOML for regular expressions. It's -the equivalent of r-strings in Python. Multiline strings are treated as verbose regular -expressions by Black. Use `[ ]` to denote a significant space character. +`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your +project. -

-Example pyproject.toml +You can find more details in our documentation: -```toml -[tool.black] -line-length = 88 -target-version = ['py37'] -include = '\.pyi?$' -extend-exclude = ''' -# A regex preceded with ^/ will apply only to files and directories -# in the root of the project. -^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) -''' -``` - -
- -### Lookup hierarchy - -Command-line options have defaults that you can see in `--help`. A `pyproject.toml` can -override those defaults. Finally, options provided by the user on the command line -override both. - -_Black_ will only ever use one `pyproject.toml` file during an entire run. It doesn't -look for multiple files, and doesn't compose configuration from different levels of the -file hierarchy. - -## Editor integration - -_Black_ can be integrated into many editors with plugins. They let you run _Black_ on -your code with the ease of doing it in your editor. To get started using _Black_ in your -editor of choice, please see -[editor_integration](https://github.com/psf/black/blob/master/docs/editor_integration.md). - -Patches are welcome for editors without an editor integration or plugin! More -information can be found in -[editor_integration](https://github.com/psf/black/blob/master/docs/editor_integration.md#other-editors). - -## blackd - -`blackd` is a small HTTP server that exposes Black's functionality over a simple -protocol. The main benefit of using it is to avoid paying the cost of starting up a new -Black process every time you want to blacken a file. Please refer to -[blackd](https://github.com/psf/black/blob/master/docs/blackd.md) to get the ball -rolling. - -## black-primer - -`black-primer` is a tool built for CI (and humans) to have _Black_ `--check` a number of -(configured in `primer.json`) Git accessible projects in parallel. -[black_primer](https://github.com/psf/black/blob/master/docs/black_primer.md) has more -information regarding its usage and configuration. - -(A PR adding Mercurial support will be accepted.) - -## Version control integration - -Use [pre-commit](https://pre-commit.com/). Once you -[have it installed](https://pre-commit.com/#install), add this to the -`.pre-commit-config.yaml` in your repository: - -```yaml -repos: - - repo: https://github.com/psf/black - rev: 20.8b1 # Replace by any tag/version: https://github.com/psf/black/tags - hooks: - - id: black - language_version: python3 # Should be a command that runs python3.6+ -``` - -Then run `pre-commit install` and you're ready to go. - -Avoid using `args` in the hook. Instead, store necessary configuration in -`pyproject.toml` so that editors and command-line usage of Black all behave consistently -for your project. See _Black_'s own -[pyproject.toml](https://github.com/psf/black/blob/master/pyproject.toml) for an -example. +- [The basics: Configuration via a file](https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file) -If you're already using Python 3.7, switch the `language_version` accordingly. Finally, -`stable` is a branch that tracks the latest release on PyPI. If you'd rather run on -master, this is also an option. +And if you're looking for more general configuration documentation: -## GitHub Actions +- [Usage and Configuration](https://black.readthedocs.io/en/stable/usage_and_configuration/index.html) -Create a file named `.github/workflows/black.yml` inside your repository with: - -```yaml -name: Lint - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: psf/black@stable -``` - -You may use `options` (Default is `'--check --diff'`) and `src` (Default is `'.'`) as -follows: - -```yaml -- uses: psf/black@stable - with: - options: "--check --verbose" - src: "./src" -``` - -## Ignoring unmodified files - -_Black_ remembers files it has already formatted, unless the `--diff` flag is used or -code is passed via standard input. This information is stored per-user. The exact -location of the file depends on the _Black_ version and the system on which _Black_ is -run. The file is non-portable. The standard location on common operating systems is: - -- Windows: - `C:\\Users\\AppData\Local\black\black\Cache\\cache...pickle` -- macOS: - `/Users//Library/Caches/black//cache...pickle` -- Linux: - `/home//.cache/black//cache...pickle` - -`file-mode` is an int flag that determines whether the file was formatted as 3.6+ only, -as .pyi, and whether string normalization was omitted. - -To override the location of these files on macOS or Linux, set the environment variable -`XDG_CACHE_HOME` to your preferred location. For example, if you want to put the cache -in the directory you're running _Black_ from, set `XDG_CACHE_HOME=.cache`. _Black_ will -then write the above files to `.cache/black//`. +**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is +"No". _Black_ is all about sensible defaults. Applying those defaults will have your +code in compliance with many other _Black_ formatted projects. ## Used by @@ -513,209 +182,39 @@ Looks like this: MIT -## Contributing to _Black_ +## Contributing -In terms of inspiration, _Black_ is about as configurable as _gofmt_. This is -deliberate. +Welcome! Happy to see you willing to make the project better. You can get started by +reading this: -Bug reports and fixes are always welcome! However, before you suggest a new feature or -configuration knob, ask yourself why you want it. If it enables better integration with -some workflow, fixes an inconsistency, speeds things up, and so on - go for it! On the -other hand, if your answer is "because I don't like a particular formatting" then you're -not ready to embrace _Black_ yet. Such changes are unlikely to get accepted. You can -still try but prepare to be disappointed. +- [Contributing: The basics](https://black.readthedocs.io/en/latest/contributing/the_basics.html) -More details can be found in -[CONTRIBUTING](https://github.com/psf/black/blob/master/CONTRIBUTING.md). +You can also take a look at the rest of the contributing docs or talk with the +developers: + +- [Contributing documentation](https://black.readthedocs.io/en/latest/contributing/index.html) +- [IRC channel on Freenode](https://webchat.freenode.net/?channels=%23blackformatter) ## Change log -The log's become rather long. It moved to its own file. +The log has become rather long. It moved to its own file. -See [CHANGES](https://github.com/psf/black/blob/master/CHANGES.md). +See [CHANGES](https://black.readthedocs.io/en/latest/change_log.html). ## Authors -Glued together by [Łukasz Langa](mailto:lukasz@langa.pl). - -Maintained with [Carol Willing](mailto:carolcode@willingconsulting.com), -[Carl Meyer](mailto:carl@oddbird.net), -[Jelle Zijlstra](mailto:jelle.zijlstra@gmail.com), -[Mika Naylor](mailto:mail@autophagy.io), -[Zsolt Dollenstein](mailto:zsol.zsol@gmail.com), -[Cooper Lees](mailto:me@cooperlees.com), and Richard Si. - -Multiple contributions by: - -- [Abdur-Rahmaan Janhangeer](mailto:arj.python@gmail.com) -- [Adam Johnson](mailto:me@adamj.eu) -- [Adam Williamson](mailto:adamw@happyassassin.net) -- [Alexander Huynh](mailto:github@grande.coffee) -- [Alex Vandiver](mailto:github@chmrr.net) -- [Allan Simon](mailto:allan.simon@supinfo.com) -- Anders-Petter Ljungquist -- [Andrew Thorp](mailto:andrew.thorp.dev@gmail.com) -- [Andrew Zhou](mailto:andrewfzhou@gmail.com) -- [Andrey](mailto:dyuuus@yandex.ru) -- [Andy Freeland](mailto:andy@andyfreeland.net) -- [Anthony Sottile](mailto:asottile@umich.edu) -- [Arjaan Buijk](mailto:arjaan.buijk@gmail.com) -- [Arnav Borbornah](mailto:arnavborborah11@gmail.com) -- [Artem Malyshev](mailto:proofit404@gmail.com) -- [Asger Hautop Drewsen](mailto:asgerdrewsen@gmail.com) -- [Augie Fackler](mailto:raf@durin42.com) -- [Aviskar KC](mailto:aviskarkc10@gmail.com) -- Batuhan Taşkaya -- [Benjamin Wohlwend](mailto:bw@piquadrat.ch) -- [Benjamin Woodruff](mailto:github@benjam.info) -- [Bharat Raghunathan](mailto:bharatraghunthan9767@gmail.com) -- [Brandt Bucher](mailto:brandtbucher@gmail.com) -- [Brett Cannon](mailto:brett@python.org) -- [Bryan Bugyi](mailto:bryan.bugyi@rutgers.edu) -- [Bryan Forbes](mailto:bryan@reigndropsfall.net) -- [Calum Lind](mailto:calumlind@gmail.com) -- [Charles](mailto:peacech@gmail.com) -- Charles Reid -- [Christian Clauss](mailto:cclauss@bluewin.ch) -- [Christian Heimes](mailto:christian@python.org) -- [Chuck Wooters](mailto:chuck.wooters@microsoft.com) -- [Chris Rose](mailto:offline@offby1.net) -- Codey Oxley -- [Cong](mailto:congusbongus@gmail.com) -- [Cooper Ry Lees](mailto:me@cooperlees.com) -- [Dan Davison](mailto:dandavison7@gmail.com) -- [Daniel Hahler](mailto:github@thequod.de) -- [Daniel M. Capella](mailto:polycitizen@gmail.com) -- Daniele Esposti -- [David Hotham](mailto:david.hotham@metaswitch.com) -- [David Lukes](mailto:dafydd.lukes@gmail.com) -- [David Szotten](mailto:davidszotten@gmail.com) -- [Denis Laxalde](mailto:denis@laxalde.org) -- [Douglas Thor](mailto:dthor@transphormusa.com) -- dylanjblack -- [Eli Treuherz](mailto:eli@treuherz.com) -- [Emil Hessman](mailto:emil@hessman.se) -- [Felix Kohlgrüber](mailto:felix.kohlgrueber@gmail.com) -- [Florent Thiery](mailto:fthiery@gmail.com) -- Francisco -- [Giacomo Tagliabue](mailto:giacomo.tag@gmail.com) -- [Greg Gandenberger](mailto:ggandenberger@shoprunner.com) -- [Gregory P. Smith](mailto:greg@krypto.org) -- Gustavo Camargo -- hauntsaninja -- [Hadi Alqattan](mailto:alqattanhadizaki@gmail.com) -- [Heaford](mailto:dan@heaford.com) -- [Hugo Barrera](mailto::hugo@barrera.io) -- Hugo van Kemenade -- [Hynek Schlawack](mailto:hs@ox.cx) -- [Ivan Katanić](mailto:ivan.katanic@gmail.com) -- [Jakub Kadlubiec](mailto:jakub.kadlubiec@skyscanner.net) -- [Jakub Warczarek](mailto:jakub.warczarek@gmail.com) -- [Jan Hnátek](mailto:jan.hnatek@gmail.com) -- [Jason Fried](mailto:me@jasonfried.info) -- [Jason Friedland](mailto:jason@friedland.id.au) -- [jgirardet](mailto:ijkl@netc.fr) -- Jim Brännlund -- [Jimmy Jia](mailto:tesrin@gmail.com) -- [Joe Antonakakis](mailto:jma353@cornell.edu) -- [Jon Dufresne](mailto:jon.dufresne@gmail.com) -- [Jonas Obrist](mailto:ojiidotch@gmail.com) -- [Jonty Wareing](mailto:jonty@jonty.co.uk) -- [Jose Nazario](mailto:jose.monkey.org@gmail.com) -- [Joseph Larson](mailto:larson.joseph@gmail.com) -- [Josh Bode](mailto:joshbode@fastmail.com) -- [Josh Holland](mailto:anowlcalledjosh@gmail.com) -- [Joshua Cannon](mailto:joshdcannon@gmail.com) -- [José Padilla](mailto:jpadilla@webapplicate.com) -- [Juan Luis Cano Rodríguez](mailto:hello@juanlu.space) -- [kaiix](mailto:kvn.hou@gmail.com) -- [Katie McLaughlin](mailto:katie@glasnt.com) -- Katrin Leinweber -- [Keith Smiley](mailto:keithbsmiley@gmail.com) -- [Kenyon Ralph](mailto:kenyon@kenyonralph.com) -- [Kevin Kirsche](mailto:Kev.Kirsche+GitHub@gmail.com) -- [Kyle Hausmann](mailto:kyle.hausmann@gmail.com) -- [Kyle Sunden](mailto:sunden@wisc.edu) -- Lawrence Chan -- [Linus Groh](mailto:mail@linusgroh.de) -- [Loren Carvalho](mailto:comradeloren@gmail.com) -- [Luka Sterbic](mailto:luka.sterbic@gmail.com) -- [LukasDrude](mailto:mail@lukas-drude.de) -- Mahmoud Hossam -- Mariatta -- [Matt VanEseltine](mailto:vaneseltine@gmail.com) -- [Matthew Clapp](mailto:itsayellow+dev@gmail.com) -- [Matthew Walster](mailto:matthew@walster.org) -- Max Smolens -- [Michael Aquilina](mailto:michaelaquilina@gmail.com) -- [Michael Flaxman](mailto:michael.flaxman@gmail.com) -- [Michael J. Sullivan](mailto:sully@msully.net) -- [Michael McClimon](mailto:michael@mcclimon.org) -- [Miguel Gaiowski](mailto:miggaiowski@gmail.com) -- [Mike](mailto:roshi@fedoraproject.org) -- [mikehoyio](mailto:mikehoy@gmail.com) -- [Min ho Kim](mailto:minho42@gmail.com) -- [Miroslav Shubernetskiy](mailto:miroslav@miki725.com) -- MomIsBestFriend -- [Nathan Goldbaum](mailto:ngoldbau@illinois.edu) -- [Nathan Hunt](mailto:neighthan.hunt@gmail.com) -- [Neraste](mailto:neraste.herr10@gmail.com) -- [Nikolaus Waxweiler](mailto:madigens@gmail.com) -- [Ofek Lev](mailto:ofekmeister@gmail.com) -- [Osaetin Daniel](mailto:osaetindaniel@gmail.com) -- [otstrel](mailto:otstrel@gmail.com) -- [Pablo Galindo](mailto:Pablogsal@gmail.com) -- [Paul Ganssle](mailto:p.ganssle@gmail.com) -- [Paul Meinhardt](mailto:mnhrdt@gmail.com) -- [Peter Bengtsson](mailto:mail@peterbe.com) -- [Peter Grayson](mailto:pete@jpgrayson.net) -- [Peter Stensmyr](mailto:peter.stensmyr@gmail.com) -- pmacosta -- [Quentin Pradet](mailto:quentin@pradet.me) -- [Ralf Schmitt](mailto:ralf@systemexit.de) -- [Ramón Valles](mailto:mroutis@protonmail.com) -- [Richard Fearn](mailto:richardfearn@gmail.com) -- Richard Si -- [Rishikesh Jha](mailto:rishijha424@gmail.com) -- [Rupert Bedford](mailto:rupert@rupertb.com) -- Russell Davis -- [Rémi Verschelde](mailto:rverschelde@gmail.com) -- [Sami Salonen](mailto:sakki@iki.fi) -- [Samuel Cormier-Iijima](mailto:samuel@cormier-iijima.com) -- [Sanket Dasgupta](mailto:sanketdasgupta@gmail.com) -- Sergi -- [Scott Stevenson](mailto:scott@stevenson.io) -- Shantanu -- [shaoran](mailto:shaoran@sakuranohana.org) -- [Shinya Fujino](mailto:shf0811@gmail.com) -- springstan -- [Stavros Korokithakis](mailto:hi@stavros.io) -- [Stephen Rosen](mailto:sirosen@globus.org) -- [Steven M. Vascellaro](mailto:S.Vascellaro@gmail.com) -- [Sunil Kapil](mailto:snlkapil@gmail.com) -- [Sébastien Eustace](mailto:sebastien.eustace@gmail.com) -- [Tal Amuyal](mailto:TalAmuyal@gmail.com) -- [Terrance](mailto:git@terrance.allofti.me) -- [Thom Lu](mailto:thomas.c.lu@gmail.com) -- [Thomas Grainger](mailto:tagrain@gmail.com) -- [Tim Gates](mailto:tim.gates@iress.com) -- [Tim Swast](mailto:swast@google.com) -- [Timo](mailto:timo_tk@hotmail.com) -- Toby Fleming -- [Tom Christie](mailto:tom@tomchristie.com) -- [Tony Narlock](mailto:tony@git-pull.com) -- [Tsuyoshi Hombashi](mailto:tsuyoshi.hombashi@gmail.com) -- [Tushar Chandra](mailto:tusharchandra2018@u.northwestern.edu) -- [Tzu-ping Chung](mailto:uranusjr@gmail.com) -- [Utsav Shah](mailto:ukshah2@illinois.edu) -- utsav-dbx -- vezeli -- [Ville Skyttä](mailto:ville.skytta@iki.fi) -- [Vishwas B Sharma](mailto:sharma.vishwas88@gmail.com) -- [Vlad Emelianov](mailto:volshebnyi@gmail.com) -- [williamfzc](mailto:178894043@qq.com) -- [wouter bolsterlee](mailto:wouter@bolsterl.ee) -- Yazdan -- [Yngve Høiseth](mailto:yngve@hoiseth.net) -- [Yurii Karabas](mailto:1998uriyyo@gmail.com) -- [Zac Hatfield-Dodds](mailto:zac@zhd.dev) +The author list is quite long nowadays, so it lives in its own file. + +See [AUTHORS.md](./AUTHORS.md) + +## Code of Conduct + +Everyone participating in the _Black_ project, and in particular in the issue tracker, +pull requests, and social media activity, is expected to treat other people with respect +and more generally to follow the guidelines articulated in the +[Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/). + +At the same time, humor is encouraged. In fact, basic familiarity with Monty Python's +Flying Circus is expected. We are not savages. + +And if you _really_ need to slap somebody, do it with a fish while dancing. diff --git a/docs/authors.md b/docs/authors.md deleted file mode 100644 index 9f2ea0571b9..00000000000 --- a/docs/authors.md +++ /dev/null @@ -1,187 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# Authors - -Glued together by [Łukasz Langa](mailto:lukasz@langa.pl). - -Maintained with [Carol Willing](mailto:carolcode@willingconsulting.com), -[Carl Meyer](mailto:carl@oddbird.net), -[Jelle Zijlstra](mailto:jelle.zijlstra@gmail.com), -[Mika Naylor](mailto:mail@autophagy.io), -[Zsolt Dollenstein](mailto:zsol.zsol@gmail.com), and -[Cooper Lees](mailto:me@cooperlees.com). - -Multiple contributions by: - -- [Abdur-Rahmaan Janhangeer](mailto:arj.python@gmail.com) -- [Adam Johnson](mailto:me@adamj.eu) -- [Adam Williamson](mailto:adamw@happyassassin.net) -- [Alexander Huynh](mailto:github@grande.coffee) -- [Alex Vandiver](mailto:github@chmrr.net) -- [Allan Simon](mailto:allan.simon@supinfo.com) -- Anders-Petter Ljungquist -- [Andrew Thorp](mailto:andrew.thorp.dev@gmail.com) -- [Andrew Zhou](mailto:andrewfzhou@gmail.com) -- [Andrey](mailto:dyuuus@yandex.ru) -- [Andy Freeland](mailto:andy@andyfreeland.net) -- [Anthony Sottile](mailto:asottile@umich.edu) -- [Arjaan Buijk](mailto:arjaan.buijk@gmail.com) -- [Arnav Borbornah](mailto:arnavborborah11@gmail.com) -- [Artem Malyshev](mailto:proofit404@gmail.com) -- [Asger Hautop Drewsen](mailto:asgerdrewsen@gmail.com) -- [Augie Fackler](mailto:raf@durin42.com) -- [Aviskar KC](mailto:aviskarkc10@gmail.com) -- Batuhan Taşkaya -- [Benjamin Wohlwend](mailto:bw@piquadrat.ch) -- [Benjamin Woodruff](mailto:github@benjam.info) -- [Bharat Raghunathan](mailto:bharatraghunthan9767@gmail.com) -- [Brandt Bucher](mailto:brandtbucher@gmail.com) -- [Brett Cannon](mailto:brett@python.org) -- [Bryan Bugyi](mailto:bryan.bugyi@rutgers.edu) -- [Bryan Forbes](mailto:bryan@reigndropsfall.net) -- [Calum Lind](mailto:calumlind@gmail.com) -- [Charles](mailto:peacech@gmail.com) -- Charles Reid -- [Christian Clauss](mailto:cclauss@bluewin.ch) -- [Christian Heimes](mailto:christian@python.org) -- [Chuck Wooters](mailto:chuck.wooters@microsoft.com) -- [Chris Rose](mailto:offline@offby1.net) -- Codey Oxley -- [Cong](mailto:congusbongus@gmail.com) -- [Cooper Ry Lees](mailto:me@cooperlees.com) -- [Dan Davison](mailto:dandavison7@gmail.com) -- [Daniel Hahler](mailto:github@thequod.de) -- [Daniel M. Capella](mailto:polycitizen@gmail.com) -- Daniele Esposti -- [David Hotham](mailto:david.hotham@metaswitch.com) -- [David Lukes](mailto:dafydd.lukes@gmail.com) -- [David Szotten](mailto:davidszotten@gmail.com) -- [Denis Laxalde](mailto:denis@laxalde.org) -- [Douglas Thor](mailto:dthor@transphormusa.com) -- dylanjblack -- [Eli Treuherz](mailto:eli@treuherz.com) -- [Emil Hessman](mailto:emil@hessman.se) -- [Felix Kohlgrüber](mailto:felix.kohlgrueber@gmail.com) -- [Florent Thiery](mailto:fthiery@gmail.com) -- Francisco -- [Giacomo Tagliabue](mailto:giacomo.tag@gmail.com) -- [Greg Gandenberger](mailto:ggandenberger@shoprunner.com) -- [Gregory P. Smith](mailto:greg@krypto.org) -- Gustavo Camargo -- hauntsaninja -- [Hadi Alqattan](mailto:alqattanhadizaki@gmail.com) -- [Heaford](mailto:dan@heaford.com) -- [Hugo Barrera](mailto::hugo@barrera.io) -- Hugo van Kemenade -- [Hynek Schlawack](mailto:hs@ox.cx) -- [Ivan Katanić](mailto:ivan.katanic@gmail.com) -- [Jakub Kadlubiec](mailto:jakub.kadlubiec@skyscanner.net) -- [Jakub Warczarek](mailto:jakub.warczarek@gmail.com) -- [Jan Hnátek](mailto:jan.hnatek@gmail.com) -- [Jason Fried](mailto:me@jasonfried.info) -- [Jason Friedland](mailto:jason@friedland.id.au) -- [jgirardet](mailto:ijkl@netc.fr) -- Jim Brännlund -- [Jimmy Jia](mailto:tesrin@gmail.com) -- [Joe Antonakakis](mailto:jma353@cornell.edu) -- [Jon Dufresne](mailto:jon.dufresne@gmail.com) -- [Jonas Obrist](mailto:ojiidotch@gmail.com) -- [Jonty Wareing](mailto:jonty@jonty.co.uk) -- [Jose Nazario](mailto:jose.monkey.org@gmail.com) -- [Joseph Larson](mailto:larson.joseph@gmail.com) -- [Josh Bode](mailto:joshbode@fastmail.com) -- [Josh Holland](mailto:anowlcalledjosh@gmail.com) -- [José Padilla](mailto:jpadilla@webapplicate.com) -- [Juan Luis Cano Rodríguez](mailto:hello@juanlu.space) -- [kaiix](mailto:kvn.hou@gmail.com) -- [Katie McLaughlin](mailto:katie@glasnt.com) -- Katrin Leinweber -- [Keith Smiley](mailto:keithbsmiley@gmail.com) -- [Kenyon Ralph](mailto:kenyon@kenyonralph.com) -- [Kevin Kirsche](mailto:Kev.Kirsche+GitHub@gmail.com) -- [Kyle Hausmann](mailto:kyle.hausmann@gmail.com) -- [Kyle Sunden](mailto:sunden@wisc.edu) -- Lawrence Chan -- [Linus Groh](mailto:mail@linusgroh.de) -- [Loren Carvalho](mailto:comradeloren@gmail.com) -- [Luka Sterbic](mailto:luka.sterbic@gmail.com) -- [LukasDrude](mailto:mail@lukas-drude.de) -- Mahmoud Hossam -- Mariatta -- [Matt VanEseltine](mailto:vaneseltine@gmail.com) -- [Matthew Clapp](mailto:itsayellow+dev@gmail.com) -- [Matthew Walster](mailto:matthew@walster.org) -- Max Smolens -- [Michael Aquilina](mailto:michaelaquilina@gmail.com) -- [Michael Flaxman](mailto:michael.flaxman@gmail.com) -- [Michael J. Sullivan](mailto:sully@msully.net) -- [Michael McClimon](mailto:michael@mcclimon.org) -- [Miguel Gaiowski](mailto:miggaiowski@gmail.com) -- [Mike](mailto:roshi@fedoraproject.org) -- [mikehoyio](mailto:mikehoy@gmail.com) -- [Min ho Kim](mailto:minho42@gmail.com) -- [Miroslav Shubernetskiy](mailto:miroslav@miki725.com) -- MomIsBestFriend -- [Nathan Goldbaum](mailto:ngoldbau@illinois.edu) -- [Nathan Hunt](mailto:neighthan.hunt@gmail.com) -- [Neraste](mailto:neraste.herr10@gmail.com) -- [Nikolaus Waxweiler](mailto:madigens@gmail.com) -- [Ofek Lev](mailto:ofekmeister@gmail.com) -- [Osaetin Daniel](mailto:osaetindaniel@gmail.com) -- [otstrel](mailto:otstrel@gmail.com) -- [Pablo Galindo](mailto:Pablogsal@gmail.com) -- [Paul Ganssle](mailto:p.ganssle@gmail.com) -- [Paul Meinhardt](mailto:mnhrdt@gmail.com) -- [Paul "TBBle" Hampson](mailto:Paul.Hampson@Pobox.com) -- [Peter Bengtsson](mailto:mail@peterbe.com) -- [Peter Grayson](mailto:pete@jpgrayson.net) -- [Peter Stensmyr](mailto:peter.stensmyr@gmail.com) -- pmacosta -- [Quentin Pradet](mailto:quentin@pradet.me) -- [Ralf Schmitt](mailto:ralf@systemexit.de) -- [Ramón Valles](mailto:mroutis@protonmail.com) -- [Richard Fearn](mailto:richardfearn@gmail.com) -- Richard Si -- [Rishikesh Jha](mailto:rishijha424@gmail.com) -- [Rupert Bedford](mailto:rupert@rupertb.com) -- Russell Davis -- [Rémi Verschelde](mailto:rverschelde@gmail.com) -- [Sami Salonen](mailto:sakki@iki.fi) -- [Samuel Cormier-Iijima](mailto:samuel@cormier-iijima.com) -- [Sanket Dasgupta](mailto:sanketdasgupta@gmail.com) -- Sergi -- [Scott Stevenson](mailto:scott@stevenson.io) -- Shantanu -- [shaoran](mailto:shaoran@sakuranohana.org) -- [Shinya Fujino](mailto:shf0811@gmail.com) -- springstan -- [Stavros Korokithakis](mailto:hi@stavros.io) -- [Stephen Rosen](mailto:sirosen@globus.org) -- [Steven M. Vascellaro](mailto:S.Vascellaro@gmail.com) -- [Sunil Kapil](mailto:snlkapil@gmail.com) -- [Sébastien Eustace](mailto:sebastien.eustace@gmail.com) -- [Tal Amuyal](mailto:TalAmuyal@gmail.com) -- [Terrance](mailto:git@terrance.allofti.me) -- [Thom Lu](mailto:thomas.c.lu@gmail.com) -- [Thomas Grainger](mailto:tagrain@gmail.com) -- [Tim Gates](mailto:tim.gates@iress.com) -- [Tim Swast](mailto:swast@google.com) -- [Timo](mailto:timo_tk@hotmail.com) -- Toby Fleming -- [Tom Christie](mailto:tom@tomchristie.com) -- [Tony Narlock](mailto:tony@git-pull.com) -- [Tsuyoshi Hombashi](mailto:tsuyoshi.hombashi@gmail.com) -- [Tushar Chandra](mailto:tusharchandra2018@u.northwestern.edu) -- [Tzu-ping Chung](mailto:uranusjr@gmail.com) -- [Utsav Shah](mailto:ukshah2@illinois.edu) -- utsav-dbx -- vezeli -- [Ville Skyttä](mailto:ville.skytta@iki.fi) -- [Vishwas B Sharma](mailto:sharma.vishwas88@gmail.com) -- [Vlad Emelianov](mailto:volshebnyi@gmail.com) -- [williamfzc](mailto:178894043@qq.com) -- [wouter bolsterlee](mailto:wouter@bolsterl.ee) -- Yazdan -- [Yngve Høiseth](mailto:yngve@hoiseth.net) -- [Yurii Karabas](mailto:1998uriyyo@gmail.com) -- [Zac Hatfield-Dodds](mailto:zac@zhd.dev) diff --git a/docs/authors.md b/docs/authors.md new file mode 120000 index 00000000000..3234d6e0792 --- /dev/null +++ b/docs/authors.md @@ -0,0 +1 @@ +../AUTHORS.md \ No newline at end of file diff --git a/docs/black_primer.md b/docs/black_primer.md deleted file mode 100644 index a2dd964b7dc..00000000000 --- a/docs/black_primer.md +++ /dev/null @@ -1,120 +0,0 @@ -# black-primer - -`black-primer` is a tool built for CI (and humans) to have _Black_ `--check` a number of -(configured in `primer.json`) Git accessible projects in parallel. _(A PR will be -accepted to add Mercurial support.)_ - -## Run flow - -- Ensure we have a `black` + `git` in PATH -- Load projects from `primer.json` -- Run projects in parallel with `--worker` workers (defaults to CPU count / 2) - - Checkout projects - - Run black and record result - - Clean up repository checkout _(can optionally be disabled via `--keep`)_ -- Display results summary to screen -- Default to cleaning up `--work-dir` (which defaults to tempfile schemantics) -- Return - - 0 for successful run - - < 0 for environment / internal error - - \> 0 for each project with an error - -## Speed up runs 🏎 - -If you're running locally yourself to test black on lots of code try: - -- Using `-k` / `--keep` + `-w` / `--work-dir` so you don't have to re-checkout the repo - each run - -## CLI arguments - -```text -Usage: black-primer [OPTIONS] - - primer - prime projects for blackening... 🏴 - -Options: - -c, --config PATH JSON config file path [default: /Users/cooper/repos/ - black/src/black_primer/primer.json] - - --debug Turn on debug logging [default: False] - -k, --keep Keep workdir + repos post run [default: False] - -L, --long-checkouts Pull big projects to test [default: False] - -R, --rebase Rebase project if already checked out [default: - False] - - -w, --workdir PATH Directory path for repo checkouts [default: /var/fol - ders/tc/hbwxh76j1hn6gqjd2n2sjn4j9k1glp/T/primer.20200 - 517125229] - - -W, --workers INTEGER Number of parallel worker coroutines [default: 2] - -h, --help Show this message and exit. -``` - -## Primer config file - -The config file is in JSON format. Its main element is the `"projects"` dictionary and -each parameter is explained below: - -```json -{ - "projects": { - "00_Example": { - "cli_arguments": "List of extra CLI arguments to pass Black for this project", - "expect_formatting_changes": "Boolean to indicate that the version of Black is expected to cause changes", - "git_clone_url": "URL you would pass `git clone` to check out this repo", - "long_checkout": "Boolean to have repo skipped by default unless `--long-checkouts` is specified", - "py_versions": "List of major Python versions to run this project with - all will do as you'd expect - run on ALL versions" - }, - "aioexabgp": { - "cli_arguments": [], - "expect_formatting_changes": true, - "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", - "long_checkout": false, - "py_versions": ["all", "3.8"] - } - } -} -``` - -An example primer config file is used by Black -[here](https://github.com/psf/black/blob/master/src/black_primer/primer.json) - -## Example run - -```console -cooper-mbp:black cooper$ ~/venvs/b/bin/black-primer -[2020-05-17 13:06:40,830] INFO: 4 projects to run Black over (lib.py:270) -[2020-05-17 13:06:44,215] INFO: Analyzing results (lib.py:285) --- primer results 📊 -- - -3 / 4 succeeded (75.0%) ✅ -1 / 4 FAILED (25.0%) 💩 - - 0 projects disabled by config - - 0 projects skipped due to Python version - - 0 skipped due to long checkout - -Failed projects: - -## flake8-bugbear: - - Returned 1 - - stdout: ---- tests/b303_b304.py 2020-05-17 20:04:09.991227 +0000 -+++ tests/b303_b304.py 2020-05-17 20:06:42.753851 +0000 -@@ -26,11 +26,11 @@ - maxint = 5 # this is okay - # the following should not crash - (a, b, c) = list(range(3)) - # it is different than this - a, b, c = list(range(3)) -- a, b, c, = list(range(3)) -+ a, b, c = list(range(3)) - # and different than this - (a, b), c = list(range(3)) - a, *b, c = [1, 2, 3, 4, 5] - b[1:3] = [0, 0] - -would reformat tests/b303_b304.py -Oh no! 💥 💔 💥 -1 file would be reformatted, 22 files would be left unchanged. -``` diff --git a/docs/conf.py b/docs/conf.py index 9e03d05e937..d5673738452 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,60 +13,11 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # from pathlib import Path -import re import string -from typing import Callable, Dict, List, Optional, Pattern, Tuple, Set -from dataclasses import dataclass -import logging from pkg_resources import get_distribution -logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) - -LOG = logging.getLogger(__name__) - CURRENT_DIR = Path(__file__).parent -README = CURRENT_DIR / ".." / "README.md" -REFERENCE_DIR = CURRENT_DIR / "reference" -STATIC_DIR = CURRENT_DIR / "_static" - - -@dataclass -class SrcRange: - """Tracks which part of a file to get a section's content. - - Data: - start_line: The line where the section starts (i.e. its sub-header) (inclusive). - end_line: The line where the section ends (usually next sub-header) (exclusive). - """ - - start_line: int - end_line: int - - -@dataclass -class DocSection: - """Tracks information about a section of documentation. - - Data: - name: The section's name. This will used to detect duplicate sections. - src: The filepath to get its contents. - processors: The processors to run before writing the section to CURRENT_DIR. - out_filename: The filename to use when writing the section to CURRENT_DIR. - src_range: The line range of SRC to gets its contents. - """ - - name: str - src: Path - src_range: SrcRange = SrcRange(0, 1_000_000) - out_filename: str = "" - processors: Tuple[Callable, ...] = () - - def get_out_filename(self) -> str: - if not self.out_filename: - return self.name + ".md" - else: - return self.out_filename def make_pypi_svg(version: str) -> None: @@ -78,131 +29,10 @@ def make_pypi_svg(version: str) -> None: f.write(svg) -def make_filename(line: str) -> str: - non_letters: Pattern = re.compile(r"[^a-z]+") - filename: str = line[3:].rstrip().lower() - filename = non_letters.sub("_", filename) - if filename.startswith("_"): - filename = filename[1:] - if filename.endswith("_"): - filename = filename[:-1] - return filename + ".md" - - -def get_contents(section: DocSection) -> str: - """Gets the contents for the DocSection.""" - contents: List[str] = [] - src: Path = section.src - start_line: int = section.src_range.start_line - end_line: int = section.src_range.end_line - with open(src, "r", encoding="utf-8") as f: - for lineno, line in enumerate(f, start=1): - if lineno >= start_line and lineno < end_line: - contents.append(line) - result = "".join(contents) - # Let's make Prettier happy with the amount of trailing newlines in the sections. - if result.endswith("\n\n"): - result = result[:-1] - if not result.endswith("\n"): - result = result + "\n" - return result - - -def get_sections_from_readme() -> List[DocSection]: - """Gets the sections from README so they can be processed by process_sections. - - It opens README and goes down line by line looking for sub-header lines which - denotes a section. Once it finds a sub-header line, it will create a DocSection - object with all of the information currently available. Then on every line, it will - track the ending line index of the section. And it repeats this for every sub-header - line it finds. - """ - sections: List[DocSection] = [] - section: Optional[DocSection] = None - with open(README, "r", encoding="utf-8") as f: - for lineno, line in enumerate(f, start=1): - if line.startswith("## "): - filename = make_filename(line) - section_name = filename[:-3] - section = DocSection( - name=str(section_name), - src=README, - src_range=SrcRange(lineno, lineno), - out_filename=filename, - processors=(fix_headers,), - ) - sections.append(section) - if section is not None: - section.src_range.end_line += 1 - return sections - - -def fix_headers(contents: str) -> str: - """Fixes the headers of sections copied from README. - - Removes one octothorpe (#) from all headers since the contents are no longer nested - in a root document (i.e. the README). - """ - lines: List[str] = contents.splitlines() - fixed_contents: List[str] = [] - for line in lines: - if line.startswith("##"): - line = line[1:] - fixed_contents.append(line + "\n") # splitlines strips the leading newlines - return "".join(fixed_contents) - - -def process_sections( - custom_sections: List[DocSection], readme_sections: List[DocSection] -) -> None: - """Reads, processes, and writes sections to CURRENT_DIR. - - For each section, the contents will be fetched, processed by processors - required by the section, and written to CURRENT_DIR. If it encounters duplicate - sections (i.e. shares the same name attribute), it will skip processing the - duplicates. - - It processes custom sections before the README generated sections so sections in the - README can be overwritten with custom options. - """ - processed_sections: Dict[str, DocSection] = {} - modified_files: Set[Path] = set() - sections: List[DocSection] = custom_sections - sections.extend(readme_sections) - for section in sections: - if section.name in processed_sections: - LOG.warning( - f"Skipping '{section.name}' from '{section.src}' as it is a duplicate" - f" of a custom section from '{processed_sections[section.name].src}'" - ) - continue - - LOG.info(f"Processing '{section.name}' from '{section.src}'") - target_path: Path = CURRENT_DIR / section.get_out_filename() - if target_path in modified_files: - LOG.warning( - f"{target_path} has been already written to, its contents will be" - " OVERWRITTEN and notices will be duplicated" - ) - contents: str = get_contents(section) - - # processors goes here - if fix_headers in section.processors: - contents = fix_headers(contents) - - with open(target_path, "w", encoding="utf-8") as f: - if section.src.suffix == ".md" and section.src != target_path: - rel = section.src.resolve().relative_to(CURRENT_DIR.parent) - f.write(f'[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM {rel}"\n\n') - f.write(contents) - processed_sections[section.name] = section - modified_files.add(target_path) - - # -- Project information ----------------------------------------------------- project = "Black" -copyright = "2020, Łukasz Langa and contributors to Black" +copyright = "2018-Present, Łukasz Langa and contributors to Black" author = "Łukasz Langa and contributors to Black" # Autopopulate version @@ -213,33 +43,7 @@ def process_sections( for sp in "abcfr": version = version.split(sp)[0] -custom_sections = [ - DocSection("the_black_code_style", CURRENT_DIR / "the_black_code_style.md"), - DocSection("editor_integration", CURRENT_DIR / "editor_integration.md"), - DocSection("blackd", CURRENT_DIR / "blackd.md"), - DocSection("black_primer", CURRENT_DIR / "black_primer.md"), - DocSection("contributing_to_black", CURRENT_DIR / ".." / "CONTRIBUTING.md"), -] - -# Sphinx complains when there is a source file that isn't referenced in any of the docs. -# Since some sections autogenerated from the README are unused warnings will appear. -# -# Sections must be listed to what their name is when passed through make_filename(). -blocklisted_sections_from_readme = { - "license", - "pragmatism", - "testimonials", - "used_by", - "change_log", -} - make_pypi_svg(release) -readme_sections = get_sections_from_readme() -readme_sections = [ - x for x in readme_sections if x.name not in blocklisted_sections_from_readme -] - -process_sections(custom_sections, readme_sections) # -- General configuration --------------------------------------------------- @@ -254,11 +58,11 @@ def process_sections( "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.napoleon", - "recommonmark", + "myst_parser", ] # If you need extensions of a certain version or higher, list them here. -needs_extensions = {"recommonmark": "0.5"} +needs_extensions = {"myst_parser": "0.13.7"} # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -286,6 +90,17 @@ def process_sections( # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +# We need headers to be linkable to so ask MyST-Parser to autogenerate anchor IDs for +# headers up to and including level 3. +myst_heading_anchors = 3 + +# Prettier support formatting some MyST syntax but not all, so let's disable the +# unsupported yet still enabled by default ones. +myst_disable_syntax = [ + "myst_block_break", + "myst_line_comment", + "math_block", +] # -- Options for HTML output ------------------------------------------------- @@ -299,7 +114,6 @@ def process_sections( "about.html", "navigation.html", "relations.html", - "sourcelink.html", "searchbox.html", ] } @@ -341,21 +155,6 @@ def process_sections( # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/docs/contributing/gauging_changes.md b/docs/contributing/gauging_changes.md new file mode 100644 index 00000000000..6b70e0bc9bc --- /dev/null +++ b/docs/contributing/gauging_changes.md @@ -0,0 +1,61 @@ +# Gauging changes + +A lot of the time, your change will affect formatting and/or performance. Quantifying +these changes is hard, so we have tooling to help make it easier. + +It's recommended you evaluate the quantifiable changes your _Black_ formatting +modification causes before submitting a PR. Think about if the change seems disruptive +enough to cause frustration to projects that are already "black formatted". + +## black-primer + +`black-primer` is a tool built for CI (and humans) to have _Black_ `--check` a number of +Git accessible projects in parallel. (configured in `primer.json`) _(A PR will be +accepted to add Mercurial support.)_ + +### Run flow + +- Ensure we have a `black` + `git` in PATH +- Load projects from `primer.json` +- Run projects in parallel with `--worker` workers (defaults to CPU count / 2) + - Checkout projects + - Run black and record result + - Clean up repository checkout _(can optionally be disabled via `--keep`)_ +- Display results summary to screen +- Default to cleaning up `--work-dir` (which defaults to tempfile schemantics) +- Return + - 0 for successful run + - \< 0 for environment / internal error + - \> 0 for each project with an error + +### Speed up runs 🏎 + +If you're running locally yourself to test black on lots of code try: + +- Using `-k` / `--keep` + `-w` / `--work-dir` so you don't have to re-checkout the repo + each run + +### CLI arguments + +```text +Usage: black-primer [OPTIONS] + + primer - prime projects for blackening... 🏴 + +Options: + -c, --config PATH JSON config file path [default: /Users/cooper/repos/ + black/src/black_primer/primer.json] + + --debug Turn on debug logging [default: False] + -k, --keep Keep workdir + repos post run [default: False] + -L, --long-checkouts Pull big projects to test [default: False] + -R, --rebase Rebase project if already checked out [default: + False] + + -w, --workdir PATH Directory path for repo checkouts [default: /var/fol + ders/tc/hbwxh76j1hn6gqjd2n2sjn4j9k1glp/T/primer.20200 + 517125229] + + -W, --workers INTEGER Number of parallel worker coroutines [default: 2] + -h, --help Show this message and exit. +``` diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst new file mode 100644 index 00000000000..68dfcd60c28 --- /dev/null +++ b/docs/contributing/index.rst @@ -0,0 +1,39 @@ +Contributing +============ + +.. toctree:: + :hidden: + + the_basics + gauging_changes + reference/reference_summary + +Welcome! Happy to see you willing to make the project better. Have you read the entire +`user documentation `_ yet? + +.. rubric:: Bird's eye view + +In terms of inspiration, *Black* is about as configurable as *gofmt*. This is +deliberate. + +Bug reports and fixes are always welcome! Please follow the +`issue template on GitHub `_ for best results. + +Before you suggest a new feature or configuration knob, ask yourself why you want it. If +it enables better integration with some workflow, fixes an inconsistency, speeds things +up, and so on - go for it! On the other hand, if your answer is "because I don't like a +particular formatting" then you're not ready to embrace *Black* yet. Such changes are +unlikely to get accepted. You can still try but prepare to be disappointed. + +.. rubric:: Contents + +This section covers the following topics: + +- :doc:`the_basics` +- :doc:`gauging_changes` +- :doc:`reference/reference_summary` + +For an overview on contributing to the *Black*, please checkout :doc:`the_basics`. + +If you need a reference of the functions, classes, etc. available to you while +developing *Black*, there's the :doc:`reference/reference_summary` docs. diff --git a/docs/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst similarity index 100% rename from docs/reference/reference_classes.rst rename to docs/contributing/reference/reference_classes.rst diff --git a/docs/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst similarity index 100% rename from docs/reference/reference_exceptions.rst rename to docs/contributing/reference/reference_exceptions.rst diff --git a/docs/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst similarity index 100% rename from docs/reference/reference_functions.rst rename to docs/contributing/reference/reference_functions.rst diff --git a/docs/reference/reference_summary.rst b/docs/contributing/reference/reference_summary.rst similarity index 51% rename from docs/reference/reference_summary.rst rename to docs/contributing/reference/reference_summary.rst index 780a4b46ed8..f6ff4681557 100644 --- a/docs/reference/reference_summary.rst +++ b/docs/contributing/reference/reference_summary.rst @@ -1,6 +1,11 @@ Developer reference =================== +.. note:: + + The documentation here is quite outdated and has been neglected. Many objects worthy + of inclusion aren't documented. Contributions are appreciated! + *Contents are subject to change.* .. toctree:: diff --git a/docs/contributing_to_black.md b/docs/contributing/the_basics.md similarity index 68% rename from docs/contributing_to_black.md rename to docs/contributing/the_basics.md index b911b465afd..461bff96505 100644 --- a/docs/contributing_to_black.md +++ b/docs/contributing/the_basics.md @@ -1,28 +1,11 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM CONTRIBUTING.md" +# The basics -# Contributing to _Black_ - -Welcome! Happy to see you willing to make the project better. Have you read the entire -[user documentation](https://black.readthedocs.io/en/latest/) yet? - -## Bird's eye view - -In terms of inspiration, _Black_ is about as configurable as _gofmt_. This is -deliberate. - -Bug reports and fixes are always welcome! Please follow the -[issue template on GitHub](https://github.com/psf/black/issues/new) for best results. - -Before you suggest a new feature or configuration knob, ask yourself why you want it. If -it enables better integration with some workflow, fixes an inconsistency, speeds things -up, and so on - go for it! On the other hand, if your answer is "because I don't like a -particular formatting" then you're not ready to embrace _Black_ yet. Such changes are -unlikely to get accepted. You can still try but prepare to be disappointed. +An overview on contributing to the _Black_ project. ## Technicalities Development on the latest version of Python is preferred. As of this writing it's 3.9. -You can use any operating system. I am using macOS myself and CentOS at work. +You can use any operating system. Install all development dependencies using: @@ -63,7 +46,7 @@ $ black-primer [-k -w /tmp/black_test_repos] ### News / Changelog Requirement `Black` has CI that will check for an entry corresponding to your PR in `CHANGES.md`. If -you feel this PR not require a changelog entry please state that in a comment and a +you feel this PR does not require a changelog entry please state that in a comment and a maintainer can add a `skip news` label to make the CI pass. Otherwise, please ensure you have a line in the following format: @@ -71,11 +54,17 @@ have a line in the following format: - `Black` is now more awesome (#X) ``` -To workout X, please use +Note that X should be your PR number, not issue number! To workout X, please use [Next PR Number](https://ichard26.github.io/next-pr-number/?owner=psf&name=black). This is not perfect but saves a lot of release overhead as now the releaser does not need to go back and workout what to add to the `CHANGES.md` for each release. +### Style Changes + +If a change would affect the advertised code style, please modify the documentation (The +_Black_ code style) to reflect that change. Patches that fix unintended bugs in +formatting don't need to be mentioned separately though. + ### Docs Testing If you make changes to docs, you can test they still build locally too. @@ -96,7 +85,7 @@ your PR. You may need to change configuration for it to pass. For more `black-primer` information visit the -[documentation](https://github.com/psf/black/blob/master/docs/black_primer.md). +[documentation](./gauging_changes.md#black-primer). ## Hygiene diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 00000000000..a509d34e903 --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,49 @@ +# Getting Started + +New to _Black_? Don't worry, you've found the perfect place to get started! + +## Do you like the _Black_ code style? + +Before using _Black_ on some of your code, it might be a good idea to first understand +how _Black_ will format your code. _Black_ isn't for everyone and you may find something +that is a dealbreaker for you personally, which is okay! The current _Black_ code style +[is described here](./the_black_code_style/current_style.md). + +## Try it out online + +Also, you can try out _Black_ online for minimal fuss on the +[Black Playground](https://black.vercel.app) generously created by José Padilla. + +## Installation + +_Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to +run, but can format Python 2 code too. Python 2 support needs the `typed_ast` +dependency, which be installed with `pip install black[python2]`. + +If you can't wait for the latest _hotness_ and want to install from GitHub, use: + +`pip install git+git://github.com/psf/black` + +## Basic usage + +To get started right away with sensible defaults: + +```sh +black {source_file_or_directory}... +``` + +You can run _Black_ as a package if running it as a script doesn't work: + +```sh +python -m black {source_file_or_directory}... +``` + +## Next steps + +Took a look at [the _Black_ code style](./the_black_code_style/current_style.md) and +tried out _Black_? Fantastic, you're ready for more. Why not explore some more on using +_Black_ by reading +[Usage and Configuration: The basics](./usage_and_configuration/the_basics.md). +Alternatively, you can check out the +[Introducing _Black_ to your project](./guides/introducing_black_to_your_project.md) +guide. diff --git a/docs/github_actions.md b/docs/github_actions.md deleted file mode 100644 index bd4680998d9..00000000000 --- a/docs/github_actions.md +++ /dev/null @@ -1,27 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# GitHub Actions - -Create a file named `.github/workflows/black.yml` inside your repository with: - -```yaml -name: Lint - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: psf/black@stable - with: - black_args: ". --check" -``` - -## Inputs - -### `black_args` - -**optional**: Black input arguments. Defaults to `. --check --diff`. diff --git a/docs/guides/index.rst b/docs/guides/index.rst new file mode 100644 index 00000000000..717c5c4d066 --- /dev/null +++ b/docs/guides/index.rst @@ -0,0 +1,14 @@ +Guides +====== + +.. toctree:: + :hidden: + + introducing_black_to_your_project + using_black_with_other_tools + +Wondering how to do something specific? You've found the right place! Listed below +are topic specific guides available: + +- :doc:`introducing_black_to_your_project` +- :doc:`using_black_with_other_tools` diff --git a/docs/guides/introducing_black_to_your_project.md b/docs/guides/introducing_black_to_your_project.md new file mode 100644 index 00000000000..71ccf7c114c --- /dev/null +++ b/docs/guides/introducing_black_to_your_project.md @@ -0,0 +1,50 @@ +# Introducing _Black_ to your project + +```{note} +This guide is incomplete. Contributions are welcomed and would be deeply +appreciated! +``` + +## Avoiding ruining git blame + +A long-standing argument against moving to automated code formatters like _Black_ is +that the migration will clutter up the output of `git blame`. This was a valid argument, +but since Git version 2.23, Git natively supports +[ignoring revisions in blame](https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revltrevgt) +with the `--ignore-rev` option. You can also pass a file listing the revisions to ignore +using the `--ignore-revs-file` option. The changes made by the revision will be ignored +when assigning blame. Lines modified by an ignored revision will be blamed on the +previous revision that modified those lines. + +So when migrating your project's code style to _Black_, reformat everything and commit +the changes (preferably in one massive commit). Then put the full 40 characters commit +identifier(s) into a file. + +```text +# Migrate code style to Black +5b4ab991dede475d393e9d69ec388fd6bd949699 +``` + +Afterwards, you can pass that file to `git blame` and see clean and meaningful blame +information. + +```console +$ git blame important.py --ignore-revs-file .git-blame-ignore-revs +7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 1) def very_important_function(text, file): +abdfd8b0 (Alice Doe 2019-09-23 11:39:32 -0400 2) text = text.lstrip() +7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 3) with open(file, "r+") as f: +7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 4) f.write(formatted) +``` + +You can even configure `git` to automatically ignore revisions listed in a file on every +call to `git blame`. + +```console +$ git config blame.ignoreRevsFile .git-blame-ignore-revs +``` + +**The one caveat is that GitHub and GitLab do not yet support ignoring revisions using +their native UI of blame.** So blame information will be cluttered with a reformatting +commit on those platforms. (If you'd like this feature, there's an open issue for +[GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423) and please let GitHub +know!) diff --git a/docs/compatible_configs.md b/docs/guides/using_black_with_other_tools.md similarity index 92% rename from docs/compatible_configs.md rename to docs/guides/using_black_with_other_tools.md index de81769f72d..4b22f670fe1 100644 --- a/docs/compatible_configs.md +++ b/docs/guides/using_black_with_other_tools.md @@ -1,4 +1,6 @@ -# _Black_ compatible configurations +# Using _Black_ with other tools + +## Black compatible configurations All of Black's changes are harmless (or at least, they should be), but a few do conflict against other tools. It is not uncommon to be using other tools alongside _Black_ like @@ -13,13 +15,13 @@ tools, using **their** supported file formats. Compatible configuration files can be [found here](https://github.com/psf/black/blob/master/docs/compatible_configs/). -## isort +### isort [isort](https://pypi.org/p/isort/) helps to sort and format imports in your Python code. _Black_ also formats imports, but in a different way from isort's defaults which leads to conflicting changes. -### Profile +#### Profile Since version 5.0.0, isort supports [profiles](https://pycqa.github.io/isort/docs/configuration/profiles/) to allow easy @@ -32,7 +34,7 @@ by isort. Below, an example for `pyproject.toml`: profile = "black" ``` -### Custom Configuration +#### Custom Configuration If you're using an isort version that is older than 5.0.0 or you have some custom configuration for _Black_, you can tweak your isort configuration to make it compatible @@ -47,12 +49,12 @@ ensure_newline_before_comments = True line_length = 88 ``` -### Why those options above? +#### Why those options above? _Black_ wraps imports that surpass `line-length` by moving identifiers into their own indented line. If that still doesn't fit the bill, it will put all of them in separate lines and put a trailing comma. A more detailed explanation of this behaviour can be -[found here](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#how-black-wraps-lines). +[found here](../the_black_code_style/current_style.md#how-black-wraps-lines). isort's default mode of wrapping imports that extend past the `line_length` limit is "Grid". @@ -90,7 +92,7 @@ works the same as with _Black_. **Please note** `ensure_newline_before_comments = True` only works since isort >= 5 but does not break older versions so you can keep it if you are running previous versions. -### Formats +#### Formats
.isort.cfg @@ -132,21 +134,21 @@ profile = black
-## Flake8 +### Flake8 [Flake8](https://pypi.org/p/flake8/) is a code linter. It warns you of syntax errors, possible bugs, stylistic errors, etc. For the most part, Flake8 follows [PEP 8](https://www.python.org/dev/peps/pep-0008/) when warning about stylistic errors. There are a few deviations that cause incompatibilities with _Black_. -### Configuration +#### Configuration ``` max-line-length = 88 extend-ignore = E203 ``` -### Why those options above? +#### Why those options above? In some cases, as determined by PEP 8, _Black_ will enforce an equal amount of whitespace around slice operators. Due to this, Flake8 will raise @@ -163,7 +165,7 @@ in your configuration. Also, as like with isort, flake8 should be configured to allow lines up to the length limit of `88`, _Black_'s default. This explains `max-line-length = 88`. -### Formats +#### Formats
.flake8 @@ -198,24 +200,24 @@ extend-ignore = E203
-## Pylint +### Pylint [Pylint](https://pypi.org/p/pylint/) is also a code linter like Flake8. It has the same checks as flake8 and more. In particular, it has more formatting checks regarding style conventions like variable naming. With so many checks, Pylint is bound to have some mixed feelings about _Black_'s formatting style. -### Configuration +#### Configuration ``` disable = C0330, C0326 max-line-length = 88 ``` -### Why those options above? +#### Why those options above? When _Black_ is folding very long expressions, the closing brackets will -[be dedented](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#how-black-wraps-lines). +[be dedented](../the_black_code_style/current_style.md#how-black-wraps-lines). ```py3 ImportantClass.important_method( @@ -223,7 +225,7 @@ ImportantClass.important_method( ) ``` -Although, this style is PEP 8 compliant, Pylint will raise +Although this style is PEP 8 compliant, Pylint will raise `C0330: Wrong hanging indentation before block (add 4 spaces)` warnings. Since _Black_ isn't configurable on this style, Pylint should be told to ignore these warnings via `disable = C0330`. @@ -234,7 +236,7 @@ warning `C0326: Bad whitespace` should be disabled using `disable = C0326`. And as usual, Pylint should be configured to only complain about lines that surpass `88` characters via `max-line-length = 88`. -### Formats +#### Formats
pylintrc diff --git a/docs/index.rst b/docs/index.rst index f03d247d949..a7a7160f71f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,12 +14,27 @@ possible. Blackened code looks the same regardless of the project you're reading. Formatting becomes transparent after a while and you can focus on the content instead. -Try it out now using the `Black Playground `_. +Try it out now using the `Black Playground `_. -.. note:: +.. admonition:: Note - this is a beta product + + *Black* is already `successfully used `_ by + many projects, small and big. *Black* has a comprehensive test suite, with efficient + parallel tests, our own auto formatting and parallel Continuous Integration runner. + However, *Black* is still beta. Things will probably be wonky for a while. This is + made explicit by the "Beta" trove classifier, as well as by the "b" in the versio + number. What this means for you is that **until the formatter becomes stable, you + should expect some formatting to change in the future**. That being said, no drastic + stylistic changes are planned, mostly responses to bug reports. - `Black is beta `_. + Also, as a safety measure which slows down processing, *Black* will check that the + reformatted code still produces a valid AST that is effectively equivalent to the + original (see the + `Pragmatism <./the_black_code_style/current_style.html#pragmatism>`_ + section for details). If you're feeling confident, use ``--fast``. +.. note:: + :doc:`Black is licensed under the MIT license `. Testimonials ------------ @@ -42,28 +57,62 @@ and `pipenv `_: *This vastly improves the formatting of our code. Thanks a ton!* + +Show your style +--------------- + +Use the badge in your project's README.md: + +.. code-block:: md + + [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + + +Using the badge in README.rst: + +.. code-block:: rst + + .. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + +Looks like this: + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + Contents -------- .. toctree:: - :maxdepth: 2 - - installation_and_usage - the_black_code_style - pyproject_toml - compatible_configs - editor_integration - blackd - black_primer - version_control_integration - github_actions - ignoring_unmodified_files - contributing_to_black - show_your_style + :maxdepth: 3 + :includehidden: + + the_black_code_style/index + +.. toctree:: + :maxdepth: 3 + :includehidden: + + getting_started + usage_and_configuration/index + integrations/index + guides/index + +.. toctree:: + :maxdepth: 3 + :includehidden: + + contributing/index change_log - reference/reference_summary authors +.. toctree:: + :hidden: + + GitHub ↪ + PyPI ↪ + IRC ↪ + Indices and tables ================== diff --git a/docs/installation_and_usage.md b/docs/installation_and_usage.md deleted file mode 100644 index fcde49f4e3a..00000000000 --- a/docs/installation_and_usage.md +++ /dev/null @@ -1,195 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# Installation and usage - -## Installation - -_Black_ can be installed by running `pip install black`. It requires Python 3.6.0+ to -run but you can reformat Python 2 code with it, too. - -### Install from GitHub - -If you can't wait for the latest _hotness_ and want to install from GitHub, use: - -`pip install git+git://github.com/psf/black` - -## Usage - -To get started right away with sensible defaults: - -```sh -black {source_file_or_directory} -``` - -You can run _Black_ as a package if running it as a script doesn't work: - -```sh -python -m black {source_file_or_directory} -``` - -## Command line options - -_Black_ doesn't provide many options. You can list them by running `black --help`: - -```text -Usage: black [OPTIONS] [SRC]... - - The uncompromising code formatter. - -Options: - -c, --code TEXT Format the code passed in as a string. - -l, --line-length INTEGER How many characters per line to allow. - [default: 88] - - -t, --target-version [py27|py33|py34|py35|py36|py37|py38|py39] - Python versions that should be supported by - Black's output. [default: per-file auto- - detection] - - --pyi Format all input files like typing stubs - regardless of file extension (useful when - piping source on standard input). - - -S, --skip-string-normalization - Don't normalize string quotes or prefixes. - -C, --skip-magic-trailing-comma - Don't use trailing commas as a reason to - split lines. - - --check Don't write the files back, just return the - status. Return code 0 means nothing would - change. Return code 1 means some files - would be reformatted. Return code 123 means - there was an internal error. - - --diff Don't write the files back, just output a - diff for each file on stdout. - - --color / --no-color Show colored diff. Only applies when - `--diff` is given. - - --fast / --safe If --fast given, skip temporary sanity - checks. [default: --safe] - - --include TEXT A regular expression that matches files and - directories that should be included on - recursive searches. An empty value means - all files are included regardless of the - name. Use forward slashes for directories - on all platforms (Windows, too). Exclusions - are calculated first, inclusions later. - [default: \.pyi?$] - - --exclude TEXT A regular expression that matches files and - directories that should be excluded on - recursive searches. An empty value means no - paths are excluded. Use forward slashes for - directories on all platforms (Windows, too). - Exclusions are calculated first, inclusions - later. [default: /(\.direnv|\.eggs|\.git|\. - hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.sv - n|_build|buck-out|build|dist)/] - - --force-exclude TEXT Like --exclude, but files and directories - matching this regex will be excluded even - when they are passed explicitly as - arguments. - - --extend-exclude TEXT Like --exclude, but adds additional files - and directories on top of the excluded - ones. (useful if you simply want to add to - the default) - - --stdin-filename TEXT The name of the file when passing it through - stdin. Useful to make sure Black will - respect --force-exclude option on some - editors that rely on using stdin. - - -q, --quiet Don't emit non-error messages to stderr. - Errors are still emitted; silence those with - 2>/dev/null. - - -v, --verbose Also emit messages to stderr about files - that were not changed or were ignored due to - exclusion patterns. - - --version Show the version and exit. - --config FILE Read configuration from FILE path. - -h, --help Show this message and exit. -``` - -_Black_ is a well-behaved Unix-style command-line tool: - -- it does nothing if no sources are passed to it; -- it will read from standard input and write to standard output if `-` is used as the - filename; -- it only outputs messages to users on standard error; -- exits with code 0 unless an internal error occurred (or `--check` was used). - -## Using _Black_ with other tools - -While _Black_ enforces formatting that conforms to PEP 8, other tools may raise warnings -about _Black_'s changes or will overwrite _Black_'s changes. A good example of this is -[isort](https://pypi.org/p/isort). Since _Black_ is barely configurable, these tools -should be configured to neither warn about nor overwrite _Black_'s changes. - -Actual details on _Black_ compatible configurations for various tools can be found in -[compatible_configs](https://github.com/psf/black/blob/master/docs/compatible_configs.md#black-compatible-configurations). - -## Migrating your code style without ruining git blame - -A long-standing argument against moving to automated code formatters like _Black_ is -that the migration will clutter up the output of `git blame`. This was a valid argument, -but since Git version 2.23, Git natively supports -[ignoring revisions in blame](https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revltrevgt) -with the `--ignore-rev` option. You can also pass a file listing the revisions to ignore -using the `--ignore-revs-file` option. The changes made by the revision will be ignored -when assigning blame. Lines modified by an ignored revision will be blamed on the -previous revision that modified those lines. - -So when migrating your project's code style to _Black_, reformat everything and commit -the changes (preferably in one massive commit). Then put the full 40 characters commit -identifier(s) into a file. - -``` -# Migrate code style to Black -5b4ab991dede475d393e9d69ec388fd6bd949699 -``` - -Afterwards, you can pass that file to `git blame` and see clean and meaningful blame -information. - -```console -$ git blame important.py --ignore-revs-file .git-blame-ignore-revs -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 1) def very_important_function(text, file): -abdfd8b0 (Alice Doe 2019-09-23 11:39:32 -0400 2) text = text.lstrip() -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 3) with open(file, "r+") as f: -7a1ae265 (John Smith 2019-04-15 15:55:13 -0400 4) f.write(formatted) -``` - -You can even configure `git` to automatically ignore revisions listed in a file on every -call to `git blame`. - -```console -$ git config blame.ignoreRevsFile .git-blame-ignore-revs -``` - -**The one caveat is that GitHub and GitLab do not yet support ignoring revisions using -their native UI of blame.** So blame information will be cluttered with a reformatting -commit on those platforms. (If you'd like this feature, there's an open issue for -[GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423) and please let GitHub -know!) - -## NOTE: This is a beta product - -_Black_ is already [successfully used](https://github.com/psf/black#used-by) by many -projects, small and big. It also sports a decent test suite. However, it is still very -new. Things will probably be wonky for a while. This is made explicit by the "Beta" -trove classifier, as well as by the "b" in the version number. What this means for you -is that **until the formatter becomes stable, you should expect some formatting to -change in the future**. That being said, no drastic stylistic changes are planned, -mostly responses to bug reports. - -Also, as a temporary safety measure, _Black_ will check that the reformatted code still -produces a valid AST that is equivalent to the original. This slows it down. If you're -feeling confident, use `--fast`. diff --git a/docs/editor_integration.md b/docs/integrations/editors.md similarity index 85% rename from docs/editor_integration.md rename to docs/integrations/editors.md index f2d21f21113..6a69e9e33fd 100644 --- a/docs/editor_integration.md +++ b/docs/integrations/editors.md @@ -4,7 +4,7 @@ Options include the following: -- [purcell/reformatter.el](https://github.com/purcell/reformatter.el) +- [wbolster/emacs-python-black](https://github.com/wbolster/emacs-python-black) - [proofit404/blacken](https://github.com/pythonic-emacs/blacken) - [Elpy](https://github.com/jorgenschaefer/elpy). @@ -16,7 +16,7 @@ Options include the following: $ pip install black ``` -2. Locate your `black` installation folder. +1. Locate your `black` installation folder. On macOS / Linux / BSD: @@ -35,7 +35,7 @@ Options include the following: Note that if you are using a virtual environment detected by PyCharm, this is an unneeded step. In this case the path to `black` is `$PyInterpreterDirectory$/black`. -3. Open External tools in PyCharm/IntelliJ IDEA +1. Open External tools in PyCharm/IntelliJ IDEA On macOS: @@ -45,29 +45,29 @@ Options include the following: `File -> Settings -> Tools -> External Tools` -4. Click the + icon to add a new external tool with the following values: +1. Click the + icon to add a new external tool with the following values: - Name: Black - Description: Black is the uncompromising Python code formatter. - - Program: + - Program: \ - Arguments: `"$FilePath$"` -5. Format the currently opened file by selecting `Tools -> External Tools -> black`. +1. Format the currently opened file by selecting `Tools -> External Tools -> black`. - Alternatively, you can set a keyboard shortcut by navigating to `Preferences or Settings -> Keymap -> External Tools -> External Tools - Black`. -6. Optionally, run _Black_ on every file save: +1. Optionally, run _Black_ on every file save: 1. Make sure you have the [File Watchers](https://plugins.jetbrains.com/plugin/7177-file-watchers) plugin installed. - 2. Go to `Preferences or Settings -> Tools -> File Watchers` and click `+` to add a + 1. Go to `Preferences or Settings -> Tools -> File Watchers` and click `+` to add a new watcher: - Name: Black - File type: Python - Scope: Project Files - - Program: + - Program: \ - Arguments: `$FilePath$` - Output paths to refresh: `$FilePath$` - Working directory: `$ProjectFileDir$` @@ -87,13 +87,13 @@ Wing supports black via the OS Commands tool, as explained in the Wing documenta $ pip install black ``` -2. Make sure it runs from the command line, e.g. +1. Make sure it runs from the command line, e.g. ```console $ black --help ``` -3. In Wing IDE, activate the **OS Commands** panel and define the command **black** to +1. In Wing IDE, activate the **OS Commands** panel and define the command **black** to execute black on the currently selected file: - Use the Tools -> OS Commands menu selection @@ -106,7 +106,7 @@ Wing supports black via the OS Commands tool, as explained in the Wing documenta - [x] Auto-save files before execution - [x] Line mode -4. Select a file in the editor and press **F1** , or whatever key binding you selected +1. Select a file in the editor and press **F1** , or whatever key binding you selected in step 3, to reformat the file. ## Vim @@ -238,8 +238,10 @@ $ pip install -U black --no-binary regex,typed-ast ### With ALE 1. Install [`ale`](https://github.com/dense-analysis/ale) -2. Install `black` -3. Add this to your vimrc: + +1. Install `black` + +1. Add this to your vimrc: ```vim let g:ale_fixers = {} @@ -256,10 +258,10 @@ $ gedit ``` 1. `Go to edit > preferences > plugins` -2. Search for `external tools` and activate it. -3. In `Tools menu -> Manage external tools` -4. Add a new tool using `+` button. -5. Copy the below content to the code window. +1. Search for `external tools` and activate it. +1. In `Tools menu -> Manage external tools` +1. Add a new tool using `+` button. +1. Copy the below content to the code window. ```console #!/bin/bash @@ -319,17 +321,3 @@ hook global WinSetOption filetype=python %{ ## Thonny Use [Thonny-black-code-format](https://github.com/Franccisco/thonny-black-code-format). - -## Other integrations - -Other editors and tools will require external contributions. - -Patches welcome! ✨ 🍰 ✨ - -Any tool that can pipe code through _Black_ using its stdio mode (just -[use `-` as the file name](https://www.tldp.org/LDP/abs/html/special-chars.html#DASHREF2)). -The formatted code will be returned on stdout (unless `--check` was passed). _Black_ -will still emit messages on stderr but that shouldn't affect your use case. - -This can be used for example with PyCharm's or IntelliJ's -[File Watchers](https://www.jetbrains.com/help/pycharm/file-watchers.html). diff --git a/docs/integrations/github_actions.md b/docs/integrations/github_actions.md new file mode 100644 index 00000000000..9e8cf436453 --- /dev/null +++ b/docs/integrations/github_actions.md @@ -0,0 +1,35 @@ +# GitHub Actions integration + +You can use _Black_ within a GitHub Actions workflow without setting your own Python +environment. Great for enforcing that your code matches the _Black_ code style. + +## Usage + +Create a file named `.github/workflows/black.yml` inside your repository with: + +```yaml +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: psf/black@stable +``` + +We recommend the use of the `@stable` tag, but per version tags also exist if you prefer +that. + +You may use `options` (Default is `'--check --diff'`) and `src` (Default is `'.'`) as +follows: + +```yaml +- uses: psf/black@stable + with: + options: "--check --verbose" + src: "./src" +``` diff --git a/docs/integrations/index.rst b/docs/integrations/index.rst new file mode 100644 index 00000000000..ed62ebcf044 --- /dev/null +++ b/docs/integrations/index.rst @@ -0,0 +1,28 @@ +Integrations +============ + +.. toctree:: + :hidden: + + editors + github_actions + source_version_control + +*Black* can be integrated into many environments, providing a better and smoother experience. Documentation for integrating *Black* with a tool can be found for the +following areas: + +- :doc:`Editor / IDE <./editors>` +- :doc:`GitHub Actions <./github_actions>` +- :doc:`Source version control <./source_version_control>` + +Editors and tools not listed will require external contributions. + +Patches welcome! ✨ 🍰 ✨ + +Any tool can pipe code through *Black* using its stdio mode (just +`use \`-\` as the file name `_). +The formatted code will be returned on stdout (unless ``--check`` was passed). *Black* +will still emit messages on stderr but that shouldn't affect your use case. + +This can be used for example with PyCharm's or IntelliJ's +`File Watchers `_. diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md new file mode 100644 index 00000000000..1ca6161bf0b --- /dev/null +++ b/docs/integrations/source_version_control.md @@ -0,0 +1,14 @@ +# Version control integration + +Use [pre-commit](https://pre-commit.com/). Once you +[have it installed](https://pre-commit.com/#install), add this to the +`.pre-commit-config.yaml` in your repository: + +```yaml +repos: + - repo: https://github.com/psf/black + rev: stable # Replace by any tag/version: https://github.com/psf/black/tags + hooks: + - id: black + language_version: python3 # Should be a command that runs python3.6+ +``` diff --git a/docs/license.rst b/docs/license.rst new file mode 100644 index 00000000000..2dc20a24492 --- /dev/null +++ b/docs/license.rst @@ -0,0 +1,6 @@ +:orphan: + +License +======= + +.. include:: ../LICENSE diff --git a/docs/pyproject_toml.md b/docs/pyproject_toml.md deleted file mode 100644 index ed88f37aa4a..00000000000 --- a/docs/pyproject_toml.md +++ /dev/null @@ -1,91 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# pyproject.toml - -_Black_ is able to read project-specific default values for its command line options -from a `pyproject.toml` file. This is especially useful for specifying custom -`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your -project. - -**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is -"No". _Black_ is all about sensible defaults. - -## What on Earth is a `pyproject.toml` file? - -[PEP 518](https://www.python.org/dev/peps/pep-0518/) defines `pyproject.toml` as a -configuration file to store build system requirements for Python projects. With the help -of tools like [Poetry](https://python-poetry.org/) or -[Flit](https://flit.readthedocs.io/en/latest/) it can fully replace the need for -`setup.py` and `setup.cfg` files. - -## Where _Black_ looks for the file - -By default _Black_ looks for `pyproject.toml` starting from the common base directory of -all files and directories passed on the command line. If it's not there, it looks in -parent directories. It stops looking when it finds the file, or a `.git` directory, or a -`.hg` directory, or the root of the file system, whichever comes first. - -If you're formatting standard input, _Black_ will look for configuration starting from -the current working directory. - -You can use a "global" configuration, stored in a specific location in your home -directory. This will be used as a fallback configuration, that is, it will be used if -and only if _Black_ doesn't find any configuration as mentioned above. Depending on your -operating system, this configuration file should be stored as: - -- Windows: `~\.black` -- Unix-like (Linux, MacOS, etc.): `$XDG_CONFIG_HOME/black` (`~/.config/black` if the - `XDG_CONFIG_HOME` environment variable is not set) - -Note that these are paths to the TOML file itself (meaning that they shouldn't be named -as `pyproject.toml`), not directories where you store the configuration. Here, `~` -refers to the path to your home directory. On Windows, this will be something like -`C:\\Users\UserName`. - -You can also explicitly specify the path to a particular file that you want with -`--config`. In this situation _Black_ will not look for any other file. - -If you're running with `--verbose`, you will see a blue message if a file was found and -used. - -Files listed within a projects `.gitignore` file will not be formatted by _Black_. - -Please note `blackd` will not use `pyproject.toml` configuration. - -## Configuration format - -As the file extension suggests, `pyproject.toml` is a -[TOML](https://github.com/toml-lang/toml) file. It contains separate sections for -different tools. _Black_ is using the `[tool.black]` section. The option keys are the -same as long names of options on the command line. - -Note that you have to use single-quoted strings in TOML for regular expressions. It's -the equivalent of r-strings in Python. Multiline strings are treated as verbose regular -expressions by Black. Use `[ ]` to denote a significant space character. - -
-Example pyproject.toml - -```toml -[tool.black] -line-length = 88 -target-version = ['py37'] -include = '\.pyi?$' -extend-exclude = ''' -# A regex preceded with ^/ will apply only to files and directories -# in the root of the project. -^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) -''' -``` - -
- -## Lookup hierarchy - -Command-line options have defaults that you can see in `--help`. A `pyproject.toml` can -override those defaults. Finally, options provided by the user on the command line -override both. - -_Black_ will only ever use one `pyproject.toml` file during an entire run. It doesn't -look for multiple files, and doesn't compose configuration from different levels of the -file hierarchy. diff --git a/docs/requirements.txt b/docs/requirements.txt index fcb6809cadc..6c34e1fefab 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ -recommonmark==0.6.0 -Sphinx==3.2.1 -Pygments==2.7.4 \ No newline at end of file +MyST-Parser==0.13.7 +Sphinx==3.5.4 diff --git a/docs/show_your_style.md b/docs/show_your_style.md deleted file mode 100644 index 67b213c3965..00000000000 --- a/docs/show_your_style.md +++ /dev/null @@ -1,19 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# Show your style - -Use the badge in your project's README.md: - -```md -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -``` - -Using the badge in README.rst: - -``` -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black -``` - -Looks like this: -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/docs/the_black_code_style.md b/docs/the_black_code_style/current_style.md similarity index 91% rename from docs/the_black_code_style.md rename to docs/the_black_code_style/current_style.md index 39a452ff9a8..7d08bc9cad5 100644 --- a/docs/the_black_code_style.md +++ b/docs/the_black_code_style/current_style.md @@ -2,11 +2,13 @@ ## Code style -_Black_ reformats entire files in place. It is not configurable. It doesn't take -previous formatting into account. It doesn't reformat blocks that start with -`# fmt: off` and end with `# fmt: on`. `# fmt: on/off` have to be on the same level of -indentation. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments -to the same effect, as a courtesy for straddling code. +_Black_ reformats entire files in place. Style configuration options are deliberately +limited and rarely added. It doesn't take previous formatting into account, except for +the magic trailing comma and preserving newlines. It doesn't reformat blocks that start +with `# fmt: off` and end with `# fmt: on`, or lines that ends with `# fmt: skip`. +`# fmt: on/off` have to be on the same level of indentation. It also recognizes +[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a +courtesy for straddling code. ### How _Black_ wraps lines @@ -75,6 +77,8 @@ def very_important_function( ... ``` +(labels/why-no-backslashes)= + _Black_ prefers parentheses over backslashes, and will remove backslashes if found. ```py3 @@ -115,29 +119,6 @@ If you're reaching for backslashes, that's a clear signal that you can do better slightly refactor your code. I hope some of the examples above show you that there are many ways in which you can do it. -However there is one exception: `with` statements using multiple context managers. -Python's grammar does not allow organizing parentheses around the series of context -managers. - -We don't want formatting like: - -```py3 -with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: - ... # nothing to split on - line too long -``` - -So _Black_ will now format it like this: - -```py3 -with \ - make_context_manager(1) as cm1, \ - make_context_manager(2) as cm2, \ - make_context_manager(3) as cm3, \ - make_context_manager(4) as cm4 \ -: - ... # backslashes and an ugly stranded colon -``` - You might have noticed that closing brackets are always dedented and that a trailing comma is always added. Such formatting produces smaller diffs; when you add or remove an element, it's always just one line. Also, having the closing bracket dedented provides a @@ -289,13 +270,13 @@ If you are adopting _Black_ in a large project with pre-existing string conventi you can pass `--skip-string-normalization` on the command line. This is meant as an adoption helper, avoid using this for new projects. -As an experimental option, _Black_ splits long strings (using parentheses where -appropriate) and merges short ones. When split, parts of f-strings that don't need -formatting are converted to plain strings. User-made splits are respected when they do -not exceed the line length limit. Line continuation backslashes are converted into -parenthesized strings. Unnecessary parentheses are stripped. To enable experimental -string processing, pass `--experimental-string-processing` on the command line. Because -the functionality is experimental, feedback and issue reports are highly encouraged! +As an experimental option (can be enabled by `--experimental-string-processing`), +_Black_ splits long strings (using parentheses where appropriate) and merges short ones. +When split, parts of f-strings that don't need formatting are converted to plain +strings. User-made splits are respected when they do not exceed the line length limit. +Line continuation backslashes are converted into parenthesized strings. Unnecessary +parentheses are stripped. Because the functionality is experimental, feedback and issue +reports are highly encouraged! _Black_ also processes docstrings. Firstly the indentation of docstrings is corrected for both quotations and the text within, although relative indentation in the text is @@ -354,7 +335,7 @@ pair of parentheses to form an atom. There are a few interesting cases: In those cases, parentheses are removed when the entire statement fits in one line, or if the inner expression doesn't have any delimiters to further split on. If there is only a single delimiter and the expression starts or ends with a bracket, the -parenthesis can also be successfully omitted since the existing bracket pair will +parentheses can also be successfully omitted since the existing bracket pair will organize the expression neatly anyway. Otherwise, the parentheses are added. Please note that _Black_ does not add or remove any additional nested parentheses that @@ -453,7 +434,7 @@ into one item per line. How do you make it stop? Just delete that trailing comma and _Black_ will collapse your collection into one line if it fits. -If you must, you can recover the behaviour of early versions of Black with the option +If you must, you can recover the behaviour of early versions of _Black_ with the option `--skip-magic-trailing-comma` / `-C`. ### r"strings" and R"strings" @@ -478,11 +459,11 @@ target. There are three limited cases in which the AST does differ: of docstrings that we're aware of sanitizes indentation and leading/trailing whitespace anyway. -2. _Black_ manages optional parentheses for some statements. In the case of the `del` +1. _Black_ manages optional parentheses for some statements. In the case of the `del` statement, presence of wrapping parentheses or lack of thereof changes the resulting AST but is semantically equivalent in the interpreter. -3. _Black_ might move comments around, which includes type comments. Those are part of +1. _Black_ might move comments around, which includes type comments. Those are part of the AST as of Python 3.8. While the tool implements a number of special cases for those comments, there is no guarantee they will remain where they were in the source. Note that this doesn't change runtime behavior of the source code. diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md new file mode 100644 index 00000000000..aca9fe04017 --- /dev/null +++ b/docs/the_black_code_style/future_style.md @@ -0,0 +1,35 @@ +# The (future of the) Black code style + +```{warning} +Changes to this document often aren't tied and don't relate to releases of +_Black_. It's recommended that you read the latest version available. +``` + +## Using backslashes for with statements + +[Backslashes are bad and should be never be used](labels/why-no-backslashes) however +there is one exception: `with` statements using multiple context managers. Before Python +3.9 Python's grammar does not allow organizing parentheses around the series of context +managers. + +We don't want formatting like: + +```py3 +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + ... # nothing to split on - line too long +``` + +So _Black_ will eventually format it like this: + +```py3 +with \ + make_context_manager(1) as cm1, \ + make_context_manager(2) as cm2, \ + make_context_manager(3) as cm3, \ + make_context_manager(4) as cm4 \ +: + ... # backslashes and an ugly stranded colon +``` + +Although when the target version is Python 3.9 or higher, _Black_ will use parentheses +instead since they're allowed in Python 3.9 and higher. diff --git a/docs/the_black_code_style/index.rst b/docs/the_black_code_style/index.rst new file mode 100644 index 00000000000..4693437be8b --- /dev/null +++ b/docs/the_black_code_style/index.rst @@ -0,0 +1,19 @@ +The Black Code Style +==================== + +.. toctree:: + :hidden: + + Current style + Future style + +*Black* is a PEP 8 compliant opinionated formatter with its own style. + +It should be noted that while keeping the style unchanged throughout releases is a +goal, the *Black* code style isn't set in stone. Sometimes it's modified in response to +user feedback or even changes to the Python language! + +Documentation for both the current and future styles can be found: + +- :doc:`current_style` +- :doc:`future_style` diff --git a/docs/blackd.md b/docs/usage_and_configuration/black_as_a_server.md similarity index 98% rename from docs/blackd.md rename to docs/usage_and_configuration/black_as_a_server.md index c8058ee7c63..0c0382b3e50 100644 --- a/docs/blackd.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -1,10 +1,10 @@ -## blackd +# Black as a server (blackd) `blackd` is a small HTTP server that exposes _Black_'s functionality over a simple protocol. The main benefit of using it is to avoid the cost of starting up a new _Black_ process every time you want to blacken a file. -### Usage +## Usage `blackd` is not packaged alongside _Black_ by default because it has additional dependencies. You will need to execute `pip install black[d]` to install it. @@ -36,7 +36,7 @@ blackd --bind-port 9090 & # or let blackd choose a port curl -s -XPOST "localhost:9090" -d "print('valid')" ``` -### Protocol +## Protocol `blackd` only accepts `POST` requests at the `/` path. The body of the request should contain the python source code to be formatted, encoded according to the `charset` field diff --git a/docs/ignoring_unmodified_files.md b/docs/usage_and_configuration/file_collection_and_discovery.md similarity index 61% rename from docs/ignoring_unmodified_files.md rename to docs/usage_and_configuration/file_collection_and_discovery.md index a915f4e8678..54c76cd9a0f 100644 --- a/docs/ignoring_unmodified_files.md +++ b/docs/usage_and_configuration/file_collection_and_discovery.md @@ -1,6 +1,11 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" +# File collection and discovery -# Ignoring unmodified files +You can directly pass _Black_ files, but you can also pass directories and _Black_ will +walk them, collecting files to format. It determines what files to format or skip +automatically using the inclusion and exclusion regexes and as well their modification +time. + +## Ignoring unmodified files _Black_ remembers files it has already formatted, unless the `--diff` flag is used or code is passed via standard input. This information is stored per-user. The exact @@ -21,3 +26,12 @@ To override the location of these files on macOS or Linux, set the environment v `XDG_CACHE_HOME` to your preferred location. For example, if you want to put the cache in the directory you're running _Black_ from, set `XDG_CACHE_HOME=.cache`. _Black_ will then write the above files to `.cache/black//`. + +## .gitignore + +If `--exclude` is not set, _Black_ will automatically ignore files and directories in +`.gitignore` file, if present. The `.gitignore` file must be in the project root to be +used and nested `.gitignore` aren't supported. + +If you want _Black_ to continue using `.gitignore` while also configuring the exclusion +rules, please use `--extend-exclude`. diff --git a/docs/usage_and_configuration/index.rst b/docs/usage_and_configuration/index.rst new file mode 100644 index 00000000000..84a9c0cb99b --- /dev/null +++ b/docs/usage_and_configuration/index.rst @@ -0,0 +1,24 @@ +Usage and Configuration +======================= + +.. toctree:: + :hidden: + + the_basics + file_collection_and_discovery + black_as_a_server + +Sometimes, running *Black* with its defaults and passing filepaths to it just won't cut +it. Passing each file using paths will become burdensome, and maybe you would like +*Black* to not touch your files and just output diffs. And yes, you *can* tweak certain +parts of *Black*'s style, but please know that configurability in this area is +purposefully limited. + +Using many of these more advanced features of *Black* will require some configuration. +Configuration that will either live on the command line or in a TOML configuration file. + +This section covers features of *Black* and configuring *Black* in detail: + +- :doc:`The basics <./the_basics>` +- :doc:`File collection and discovery ` +- :doc:`Black as a server (blackd) <./black_as_a_server>` diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md new file mode 100644 index 00000000000..6fd8769b591 --- /dev/null +++ b/docs/usage_and_configuration/the_basics.md @@ -0,0 +1,227 @@ +# The basics + +Foundational knowledge on using and configuring Black. + +_Black_ is a well-behaved Unix-style command-line tool: + +- it does nothing if no sources are passed to it; +- it will read from standard input and write to standard output if `-` is used as the + filename; +- it only outputs messages to users on standard error; +- exits with code 0 unless an internal error occurred (or `--check` was used). + +## Usage + +To get started right away with sensible defaults: + +```sh +black {source_file_or_directory} +``` + +You can run _Black_ as a package if running it as a script doesn't work: + +```sh +python -m black {source_file_or_directory} +``` + +### Command line options + +_Black_ has quite a few knobs these days, although _Black_ is opinionated so style +configuration options are deliberately limited and rarely added. You can list them by +running `black --help`. + +
+ +Help output + +``` + Usage: black [OPTIONS] [SRC]... + + The uncompromising code formatter. + + Options: + -c, --code TEXT Format the code passed in as a string. + -l, --line-length INTEGER How many characters per line to allow. + [default: 88] + + -t, --target-version [py27|py33|py34|py35|py36|py37|py38|py39] + Python versions that should be supported by + Black's output. [default: per-file auto- + detection] + + --pyi Format all input files like typing stubs + regardless of file extension (useful when + piping source on standard input). + + -S, --skip-string-normalization + Don't normalize string quotes or prefixes. + -C, --skip-magic-trailing-comma + Don't use trailing commas as a reason to + split lines. + + --check Don't write the files back, just return the + status. Return code 0 means nothing would + change. Return code 1 means some files + would be reformatted. Return code 123 means + there was an internal error. + + --diff Don't write the files back, just output a + diff for each file on stdout. + + --color / --no-color Show colored diff. Only applies when + `--diff` is given. + + --fast / --safe If --fast given, skip temporary sanity + checks. [default: --safe] + + --include TEXT A regular expression that matches files and + directories that should be included on + recursive searches. An empty value means + all files are included regardless of the + name. Use forward slashes for directories + on all platforms (Windows, too). Exclusions + are calculated first, inclusions later. + [default: \.pyi?$] + + --exclude TEXT A regular expression that matches files and + directories that should be excluded on + recursive searches. An empty value means no + paths are excluded. Use forward slashes for + directories on all platforms (Windows, too). + Exclusions are calculated first, inclusions + later. [default: /(\.direnv|\.eggs|\.git|\. + hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_bu + ild|buck-out|build|dist)/] + + --extend-exclude TEXT Like --exclude, but adds additional files + and directories on top of the excluded + ones (useful if you simply want to add to + the default). + + --force-exclude TEXT Like --exclude, but files and directories + matching this regex will be excluded even + when they are passed explicitly as + arguments. + + + --stdin-filename TEXT The name of the file when passing it through + stdin. Useful to make sure Black will + respect --force-exclude option on some + editors that rely on using stdin. + + -q, --quiet Don't emit non-error messages to stderr. + Errors are still emitted; silence those with + 2>/dev/null. + + -v, --verbose Also emit messages to stderr about files + that were not changed or were ignored due to + exclusion patterns. + + --version Show the version and exit. + --config FILE Read configuration from FILE path. + -h, --help Show this message and exit. +``` + +
+ +## Configuration via a file + +_Black_ is able to read project-specific default values for its command line options +from a `pyproject.toml` file. This is especially useful for specifying custom +`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your +project. + +**Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is +"No". _Black_ is all about sensible defaults. Applying those defaults will have your +code in compliance with many other _Black_ formatted projects. + +### What on Earth is a `pyproject.toml` file? + +[PEP 518](https://www.python.org/dev/peps/pep-0518/) defines `pyproject.toml` as a +configuration file to store build system requirements for Python projects. With the help +of tools like [Poetry](https://python-poetry.org/) or +[Flit](https://flit.readthedocs.io/en/latest/) it can fully replace the need for +`setup.py` and `setup.cfg` files. + +### Where _Black_ looks for the file + +By default _Black_ looks for `pyproject.toml` starting from the common base directory of +all files and directories passed on the command line. If it's not there, it looks in +parent directories. It stops looking when it finds the file, or a `.git` directory, or a +`.hg` directory, or the root of the file system, whichever comes first. + +If you're formatting standard input, _Black_ will look for configuration starting from +the current working directory. + +You can use a "global" configuration, stored in a specific location in your home +directory. This will be used as a fallback configuration, that is, it will be used if +and only if _Black_ doesn't find any configuration as mentioned above. Depending on your +operating system, this configuration file should be stored as: + +- Windows: `~\.black` +- Unix-like (Linux, MacOS, etc.): `$XDG_CONFIG_HOME/black` (`~/.config/black` if the + `XDG_CONFIG_HOME` environment variable is not set) + +Note that these are paths to the TOML file itself (meaning that they shouldn't be named +as `pyproject.toml`), not directories where you store the configuration. Here, `~` +refers to the path to your home directory. On Windows, this will be something like +`C:\\Users\UserName`. + +You can also explicitly specify the path to a particular file that you want with +`--config`. In this situation _Black_ will not look for any other file. + +If you're running with `--verbose`, you will see a blue message if a file was found and +used. + +Please note `blackd` will not use `pyproject.toml` configuration. + +### Configuration format + +As the file extension suggests, `pyproject.toml` is a +[TOML](https://github.com/toml-lang/toml) file. It contains separate sections for +different tools. _Black_ is using the `[tool.black]` section. The option keys are the +same as long names of options on the command line. + +Note that you have to use single-quoted strings in TOML for regular expressions. It's +the equivalent of r-strings in Python. Multiline strings are treated as verbose regular +expressions by Black. Use `[ ]` to denote a significant space character. + +
+Example pyproject.toml + +```toml +[tool.black] +line-length = 88 +target-version = ['py37'] +include = '\.pyi?$' +extend-exclude = ''' +# A regex preceded with ^/ will apply only to files and directories +# in the root of the project. +^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) +''' +``` + +
+ +### Lookup hierarchy + +Command-line options have defaults that you can see in `--help`. A `pyproject.toml` can +override those defaults. Finally, options provided by the user on the command line +override both. + +_Black_ will only ever use one `pyproject.toml` file during an entire run. It doesn't +look for multiple files, and doesn't compose configuration from different levels of the +file hierarchy. + +## Next steps + +You've probably noted that not all of the options you can pass to _Black_ have been +covered. Don't worry, the rest will be covered in a later section. + +A good next step would be configuring auto-discovery so `black .` is all you need +instead of laborously listing every file or directory. You can get started by heading +over to [File collection and discovery](./file_collection_and_discovery.md). + +Another good choice would be setting up an +[integration with your editor](../integrations/editors.md) of choice or with +[pre-commit for source version control](../integrations/source_version_control.md). diff --git a/docs/version_control_integration.md b/docs/version_control_integration.md deleted file mode 100644 index 2d8bc172eba..00000000000 --- a/docs/version_control_integration.md +++ /dev/null @@ -1,28 +0,0 @@ -[//]: # "NOTE: THIS FILE WAS AUTOGENERATED FROM README.md" - -# Version control integration - -Use [pre-commit](https://pre-commit.com/). Once you -[have it installed](https://pre-commit.com/#install), add this to the -`.pre-commit-config.yaml` in your repository: - -```yaml -repos: - - repo: https://github.com/psf/black - rev: 20.8b1 # Replace by any tag/version: https://github.com/psf/black/tags - hooks: - - id: black - language_version: python3 # Should be a command that runs python3.6+ -``` - -Then run `pre-commit install` and you're ready to go. - -Avoid using `args` in the hook. Instead, store necessary configuration in -`pyproject.toml` so that editors and command-line usage of Black all behave consistently -for your project. See _Black_'s own -[pyproject.toml](https://github.com/psf/black/blob/master/pyproject.toml) for an -example. - -If you're already using Python 3.7, switch the `language_version` accordingly. Finally, -`stable` is a branch that tracks the latest release on PyPI. If you'd rather run on -master, this is also an option. From 1bedc176d19f0e71c24149c696c5a2a71f21e941 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sat, 8 May 2021 15:53:01 -0400 Subject: [PATCH 62/81] Fix autodoc refs broken by refactor (#2207) --- .../reference/reference_classes.rst | 6 +- .../reference/reference_exceptions.rst | 2 +- .../reference/reference_functions.rst | 112 +++++++++--------- 3 files changed, 59 insertions(+), 61 deletions(-) diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst index 8a2ded9dfec..fa765961e69 100644 --- a/docs/contributing/reference/reference_classes.rst +++ b/docs/contributing/reference/reference_classes.rst @@ -8,7 +8,7 @@ :class:`BracketTracker` ----------------------- -.. autoclass:: black.BracketTracker +.. autoclass:: black.brackets.BracketTracker :members: :class:`EmptyLineTracker` @@ -34,7 +34,7 @@ :class:`ProtoComment` --------------------- -.. autoclass:: black.ProtoComment +.. autoclass:: black.comments.ProtoComment :members: :class:`Report` @@ -47,7 +47,7 @@ :class:`Visitor` ---------------- -.. autoclass:: black.Visitor +.. autoclass:: black.nodes.Visitor :show-inheritance: :members: diff --git a/docs/contributing/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst index 517249fa4ce..aafe61e5017 100644 --- a/docs/contributing/reference/reference_exceptions.rst +++ b/docs/contributing/reference/reference_exceptions.rst @@ -5,7 +5,7 @@ .. currentmodule:: black -.. autoexception:: black.CannotSplit +.. autoexception:: black.linegen.CannotSplit .. autoexception:: black.NothingChanged diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst index bffce7e9e32..4353d1bf9a9 100644 --- a/docs/contributing/reference/reference_functions.rst +++ b/docs/contributing/reference/reference_functions.rst @@ -12,31 +12,31 @@ Assertions and checks .. autofunction:: black.assert_stable -.. autofunction:: black.can_be_split +.. autofunction:: black.lines.can_be_split -.. autofunction:: black.can_omit_invisible_parens +.. autofunction:: black.lines.can_omit_invisible_parens -.. autofunction:: black.is_empty_tuple +.. autofunction:: black.nodes.is_empty_tuple -.. autofunction:: black.is_import +.. autofunction:: black.nodes.is_import -.. autofunction:: black.is_line_short_enough +.. autofunction:: black.lines.is_line_short_enough -.. autofunction:: black.is_multiline_string +.. autofunction:: black.nodes.is_multiline_string -.. autofunction:: black.is_one_tuple +.. autofunction:: black.nodes.is_one_tuple -.. autofunction:: black.is_split_after_delimiter +.. autofunction:: black.brackets.is_split_after_delimiter -.. autofunction:: black.is_split_before_delimiter +.. autofunction:: black.brackets.is_split_before_delimiter -.. autofunction:: black.is_stub_body +.. autofunction:: black.nodes.is_stub_body -.. autofunction:: black.is_stub_suite +.. autofunction:: black.nodes.is_stub_suite -.. autofunction:: black.is_vararg +.. autofunction:: black.nodes.is_vararg -.. autofunction:: black.is_yield +.. autofunction:: black.nodes.is_yield Formatting @@ -70,111 +70,109 @@ Parsing .. autofunction:: black.decode_bytes -.. autofunction:: black.lib2to3_parse +.. autofunction:: black.parsing.lib2to3_parse -.. autofunction:: black.lib2to3_unparse +.. autofunction:: black.parsing.lib2to3_unparse Split functions --------------- -.. autofunction:: black.bracket_split_build_line +.. autofunction:: black.linegen.bracket_split_build_line -.. autofunction:: black.bracket_split_succeeded_or_raise +.. autofunction:: black.linegen.bracket_split_succeeded_or_raise -.. autofunction:: black.delimiter_split +.. autofunction:: black.linegen.delimiter_split -.. autofunction:: black.left_hand_split +.. autofunction:: black.linegen.left_hand_split -.. autofunction:: black.right_hand_split +.. autofunction:: black.linegen.right_hand_split -.. autofunction:: black.standalone_comment_split +.. autofunction:: black.linegen.standalone_comment_split -.. autofunction:: black.transform_line +.. autofunction:: black.linegen.transform_line Caching ------- -.. autofunction:: black.filter_cached +.. autofunction:: black.cache.filter_cached -.. autofunction:: black.get_cache_file +.. autofunction:: black.cache.get_cache_file -.. autofunction:: black.get_cache_info +.. autofunction:: black.cache.get_cache_info -.. autofunction:: black.read_cache +.. autofunction:: black.cache.read_cache -.. autofunction:: black.write_cache +.. autofunction:: black.cache.write_cache Utilities --------- -.. py:function:: black.DebugVisitor.show(code: str) -> None +.. py:function:: black.debug.DebugVisitor.show(code: str) -> None Pretty-print the lib2to3 AST of a given string of `code`. -.. autofunction:: black.cancel +.. autofunction:: black.concurrency.cancel -.. autofunction:: black.child_towards +.. autofunction:: black.nodes.child_towards -.. autofunction:: black.container_of +.. autofunction:: black.nodes.container_of -.. autofunction:: black.convert_one_fmt_off_pair +.. autofunction:: black.comments.convert_one_fmt_off_pair .. autofunction:: black.diff -.. autofunction:: black.dont_increase_indentation +.. autofunction:: black.linegen.dont_increase_indentation -.. autofunction:: black.format_float_or_int_string +.. autofunction:: black.numerics.format_float_or_int_string -.. autofunction:: black.ensure_visible +.. autofunction:: black.nodes.ensure_visible -.. autofunction:: black.enumerate_reversed +.. autofunction:: black.lines.enumerate_reversed -.. autofunction:: black.enumerate_with_length +.. autofunction:: black.comments.generate_comments -.. autofunction:: black.generate_comments +.. autofunction:: black.comments.generate_ignored_nodes -.. autofunction:: black.generate_ignored_nodes +.. autofunction:: black.comments.is_fmt_on -.. autofunction:: black.is_fmt_on +.. autofunction:: black.comments.contains_fmt_on_at_column -.. autofunction:: black.contains_fmt_on_at_column +.. autofunction:: black.nodes.first_leaf_column -.. autofunction:: black.first_leaf_column - -.. autofunction:: black.generate_trailers_to_omit +.. autofunction:: black.linegen.generate_trailers_to_omit .. autofunction:: black.get_future_imports -.. autofunction:: black.list_comments +.. autofunction:: black.comments.list_comments -.. autofunction:: black.make_comment +.. autofunction:: black.comments.make_comment -.. autofunction:: black.maybe_make_parens_invisible_in_atom +.. autofunction:: black.linegen.maybe_make_parens_invisible_in_atom -.. autofunction:: black.max_delimiter_priority_in_atom +.. autofunction:: black.brackets.max_delimiter_priority_in_atom .. autofunction:: black.normalize_fmt_off -.. autofunction:: black.normalize_numeric_literal +.. autofunction:: black.numerics.normalize_numeric_literal -.. autofunction:: black.normalize_prefix +.. autofunction:: black.linegen.normalize_prefix -.. autofunction:: black.normalize_string_prefix +.. autofunction:: black.strings.normalize_string_prefix -.. autofunction:: black.normalize_string_quotes +.. autofunction:: black.strings.normalize_string_quotes -.. autofunction:: black.normalize_invisible_parens +.. autofunction:: black.linegen.normalize_invisible_parens .. autofunction:: black.patch_click -.. autofunction:: black.preceding_leaf +.. autofunction:: black.nodes.preceding_leaf .. autofunction:: black.re_compile_maybe_verbose -.. autofunction:: black.should_split_line +.. autofunction:: black.linegen.should_split_line .. autofunction:: black.shutdown -.. autofunction:: black.sub_twice +.. autofunction:: black.strings.sub_twice -.. autofunction:: black.whitespace +.. autofunction:: black.nodes.whitespace From 5020577788eeab758a2c33d1d563e4ea82617ed4 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 9 May 2021 00:08:03 -0700 Subject: [PATCH 63/81] Remove docker CI from look at 'master' branch (#2209) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7eaf233ab9c..788c505ee4a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ name: docker on: push: branches: - - "master" + - "main" release: types: created From 06ccb88bf2bd35a4dc5d591bb296b5b299d07323 Mon Sep 17 00:00:00 2001 From: Panagiotis Vasilopoulos Date: Sun, 9 May 2021 21:50:17 +0000 Subject: [PATCH 64/81] Replace references to master branch (#2210) Commit history before merge: * Replace references to master branch * Update .flake8 to reference docs on RTD We're moving away from GitHub as a documentation host to only RTD because it's makes our lives easier creating good docs. I know this link is dead right now, but it won't be once we release a new version with the documentation reorganization changes (which should be soon!). Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- .flake8 | 2 +- .github/ISSUE_TEMPLATE/bug_report.md | 8 ++++---- CHANGES.md | 4 ++++ README.md | 6 +++--- docs/contributing/the_basics.md | 2 +- docs/guides/using_black_with_other_tools.md | 2 +- docs/integrations/editors.md | 2 +- gallery/gallery.py | 8 ++++---- setup.py | 2 +- 9 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.flake8 b/.flake8 index 656c0df24ee..68d15015a37 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] extend-ignore = E203, E266, E501 # line length is intentionally set to 80 here because black uses Bugbear -# See https://github.com/psf/black/blob/master/docs/the_black_code_style.md#line-length for more details +# See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length for more details max-line-length = 80 max-complexity = 18 select = B,C,E,F,W,T4,B9 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 3b59906594a..c6c80be5345 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -18,13 +18,13 @@ assignees: "" **Environment (please complete the following information):** -- Version: \[e.g. master\] +- Version: \[e.g. main\] - OS and Python version: \[e.g. Linux/Python 3.7.4rc1\] -**Does this bug also happen on master?** To answer this, you have two options: +**Does this bug also happen on main?** To answer this, you have two options: -1. Use the online formatter at , which will - use the latest master branch. +1. Use the online formatter at , which will use + the latest main branch. 1. Or run _Black_ on your machine: - create a new virtualenv (make sure it's the same Python version); - clone this repository; diff --git a/CHANGES.md b/CHANGES.md index 9b155924b7d..32c2d8d7237 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,10 @@ ### Documentation +- Replaced all remaining references to the + [`master`](https://github.com/psf/black/tree/main) branch with the + [`main`](https://github.com/psf/black/tree/main) branch. Some additional changes in + the source code were also made. (#2210) - Sigificantly reorganized the documentation to make much more sense. Check them out by heading over to [the stable docs on RTD](https://black.readthedocs.io/en/stable/). (#2174) diff --git a/README.md b/README.md index ceccbb79962..beb8069c32b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Black Logo](https://raw.githubusercontent.com/psf/black/master/docs/_static/logo2-readme.png) +![Black Logo](https://raw.githubusercontent.com/psf/black/main/docs/_static/logo2-readme.png)

The Uncompromising Code Formatter

@@ -6,8 +6,8 @@ Actions Status Actions Status Documentation Status -Coverage Status -License: MIT +Coverage Status +License: MIT PyPI Downloads conda-forge diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index 461bff96505..d36b17eff31 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -81,7 +81,7 @@ $ sphinx-build -a -b html -W docs/ docs/_build/ if we get source code changes. It will error on formatting changes or errors. Please run before pushing your PR to see if you get the actions you would expect from _Black_ with your PR. You may need to change -[primer.json](https://github.com/psf/black/blob/master/src/black_primer/primer.json) +[primer.json](https://github.com/psf/black/blob/main/src/black_primer/primer.json) configuration for it to pass. For more `black-primer` information visit the diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index 4b22f670fe1..2b3855f7ef0 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -13,7 +13,7 @@ tools out there. tools, using **their** supported file formats. Compatible configuration files can be -[found here](https://github.com/psf/black/blob/master/docs/compatible_configs/). +[found here](https://github.com/psf/black/blob/main/docs/compatible_configs/). ### isort diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 6a69e9e33fd..88176e1aecb 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -166,7 +166,7 @@ automatically installs _Black_. You can upgrade it later by calling `:BlackUpgra restarting Vim. If you need to do anything special to make your virtualenv work and install _Black_ (for -example you want to run a version from master), create a virtualenv manually and point +example you want to run a version from main), create a virtualenv manually and point `g:black_virtualenv` to it. The plugin will use it. To run _Black_ on save, add the following line to `.vimrc` or `init.vim`: diff --git a/gallery/gallery.py b/gallery/gallery.py index 6b42ec3a6d4..3df05c1a722 100755 --- a/gallery/gallery.py +++ b/gallery/gallery.py @@ -74,7 +74,7 @@ def get_top_packages(days: Days) -> List[str]: def get_package_source(package: str, version: Optional[str]) -> str: if package == "cpython": if version is None: - version = "master" + version = "main" return f"https://github.com/python/cpython/archive/{version}.zip" elif package == "pypy": if version is None: @@ -248,9 +248,9 @@ def format_repos(repos: Tuple[Path, ...], options: Namespace) -> None: black_version=black_version, input_directory=options.input, ) - git_switch_branch("master", repo=repo) + git_switch_branch("main", repo=repo) - git_switch_branch("master", repo=options.black_repo) + git_switch_branch("main", repo=options.black_repo) def main() -> None: @@ -296,7 +296,7 @@ def main() -> None: type=Path, help="Output directory to download and put result artifacts.", ) - parser.add_argument("versions", nargs="*", default=("master",), help="") + parser.add_argument("versions", nargs="*", default=("main",), help="") options = parser.parse_args() repos = init_repos(options) diff --git a/setup.py b/setup.py index af93d0f453a..3dc52b26e81 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def get_long_description() -> str: author="Łukasz Langa", author_email="lukasz@langa.pl", url="https://github.com/psf/black", - project_urls={"Changelog": "https://github.com/psf/black/blob/master/CHANGES.md"}, + project_urls={"Changelog": "https://github.com/psf/black/blob/main/CHANGES.md"}, license="MIT", py_modules=["_black_version"], ext_modules=ext_modules, From 3d96b7f10a56fcf826693e98f08b673dad8ac256 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sun, 9 May 2021 22:35:56 -0400 Subject: [PATCH 65/81] Autogenerate black(d|-primer)? help in usage docs (#2212) So these won't go out of date. This does mean the environment has be setup a bit more carefully so the right version of the tool is used, but thankfully the build environment is rebuilt on change on RTD anyway. Also since the HTML docs are known to build fine, let's provide downloadable HTMLzips of our docs. This change needs RTD and GH to install Black with the [d] extra so blackd's help can generated. While editing RTD's config file, let's migrate the file to a non-deprecated filename. Also I missed adding AUTHORS.md to the files key in the doc GHA config. --- .github/workflows/doc.yml | 6 +- readthedocs.yml => .readthedocs.yaml | 9 +- Pipfile | 1 + Pipfile.lock | 90 ++++++++++--------- docs/conf.py | 1 + docs/contributing/gauging_changes.md | 21 +---- docs/requirements.txt | 3 +- .../black_as_a_server.md | 10 +-- docs/usage_and_configuration/the_basics.md | 88 +----------------- 9 files changed, 70 insertions(+), 159 deletions(-) rename readthedocs.yml => .readthedocs.yaml (55%) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 930b6d440ff..74ec316a7cf 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -7,12 +7,14 @@ on: - "README.md" - "CHANGES.md" - "CONTRIBUTING.md" + - "AUTHORS.md" pull_request: paths: - "docs/**" - "README.md" - "CHANGES.md" - "CONTRIBUTING.md" + - "AUTHORS.md" jobs: build: @@ -35,8 +37,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install -e "." + python -m pip install -e ".[d]" python -m pip install -r "docs/requirements.txt" - name: Build documentation - run: sphinx-build -a -b html -W docs/ docs/_build/ + run: sphinx-build -a -b html -W --keep-going docs/ docs/_build diff --git a/readthedocs.yml b/.readthedocs.yaml similarity index 55% rename from readthedocs.yml rename to .readthedocs.yaml index 15065033d0f..24eb3eaf6d9 100644 --- a/readthedocs.yml +++ b/.readthedocs.yaml @@ -1,7 +1,14 @@ version: 2 + +formats: + - htmlzip + python: version: 3.8 install: - requirements: docs/requirements.txt - - method: setuptools + + - method: pip path: . + extra_requirements: + - d diff --git a/Pipfile b/Pipfile index d1842ff749b..b3859571137 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ mypy = ">=0.812" pre-commit = "*" readme_renderer = "*" MyST-Parser = ">=0.13.7" +sphinxcontrib-programoutput = ">=0.17" setuptools = ">=39.2.0" setuptools-scm = "*" twine = ">=1.11.0" diff --git a/Pipfile.lock b/Pipfile.lock index f973c8423e8..0b9e529e73b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "62bc4bdb0117234d1f374b2dc0685369f6df7c7192d16409cc9c42a429770166" + "sha256": "fdfbacb362e4514e588736a7e8783837cb5e3aa2fbab98ea17894fdb50d51a8e" }, "pipfile-spec": 6, "requires": {}, @@ -83,11 +83,11 @@ }, "attrs": { "hashes": [ - "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", - "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", + "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.2.0" }, "black": { "editable": true, @@ -290,13 +290,13 @@ }, "typing-extensions": { "hashes": [ - "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", - "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", - "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", "python_version <": "3.8", - "version": "==3.7.4.3", + "version": "==3.10.0.0", "version >=": "3.7.4" }, "yarl": { @@ -420,11 +420,11 @@ }, "attrs": { "hashes": [ - "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", - "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", + "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.2.0" }, "babel": { "hashes": [ @@ -631,11 +631,11 @@ }, "flake8": { "hashes": [ - "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378", - "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a" + "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b", + "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907" ], "index": "pypi", - "version": "==3.9.1" + "version": "==3.9.2" }, "flake8-bugbear": { "hashes": [ @@ -703,11 +703,11 @@ }, "markdown-it-py": { "hashes": [ - "sha256:30b3e9f8198dc82a5df0dcb73fd31d56cd9a43bf8a747feb10b2ba74f962bcb1", - "sha256:c3b9f995be0792cbbc8ab2f53d74072eb7ff8a8b622be8d61d38ab879709eca3" + "sha256:36be6bb3ad987bfdb839f5ba78ddf094552ca38ccbd784ae4f74a4e1419fc6e3", + "sha256:98080fc0bc34c4f2bcf0846a096a9429acbd9d5d8e67ed34026c03c61c464389" ], "markers": "python_version ~= '3.6'", - "version": "==0.6.2" + "version": "==1.1.0" }, "markupsafe": { "hashes": [ @@ -776,11 +776,11 @@ }, "mdit-py-plugins": { "hashes": [ - "sha256:1e467ca2ea056e8065cbd5d6c61e5052bb50826bde84c40f6a5ed77e82125710", - "sha256:77fd75dad81109ee91f30eb49146196f79afbbae041f298ae4886c8c2b5e23d7" + "sha256:1833bf738e038e35d89cb3a07eb0d227ed647ce7dd357579b65343740c6d249c", + "sha256:5991cef645502e80a5388ec4fc20885d2313d4871e8b8e320ca2de14ac0c015f" ], "markers": "python_version ~= '3.6'", - "version": "==0.2.6" + "version": "==0.2.8" }, "multidict": { "hashes": [ @@ -863,11 +863,11 @@ }, "myst-parser": { "hashes": [ - "sha256:260355b4da8e8865fe080b0638d7f1ab1791dc4bed02a7a48630b6bad4249219", - "sha256:e4bc99e43e19f70d22e528de8e7cce59f7e8e7c4c34dcba203de92de7a7c7c85" + "sha256:8d7db76e2f33cd1dc1fe0c76af9f09e5cf19ce2c2e85074bc82f272c0f7c08ce", + "sha256:fc262959a74cdc799d7fa9b30c320c17187485b9a1e8c39e988fc12f3adff63c" ], "index": "pypi", - "version": "==0.13.7" + "version": "==0.14.0" }, "nodeenv": { "hashes": [ @@ -933,11 +933,11 @@ }, "pygments": { "hashes": [ - "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94", - "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8" + "sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f", + "sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e" ], "markers": "python_version >= '3.5'", - "version": "==2.8.1" + "version": "==2.9.0" }, "pyparsing": { "hashes": [ @@ -1061,10 +1061,10 @@ }, "rfc3986": { "hashes": [ - "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d", - "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50" + "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835", + "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97" ], - "version": "==1.4.0" + "version": "==1.5.0" }, "secretstorage": { "hashes": [ @@ -1084,11 +1084,11 @@ }, "six": { "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.15.0" + "version": "==1.16.0" }, "snowballstemmer": { "hashes": [ @@ -1137,6 +1137,14 @@ "markers": "python_version >= '3.5'", "version": "==1.0.1" }, + "sphinxcontrib-programoutput": { + "hashes": [ + "sha256:0ef1c1d9159dbe7103077748214305eb4e0138e861feb71c0c346afc5fe97f84", + "sha256:300ee9b8caee8355d25cc74b4d1c7efd12e608d2ad165e3141d31e6fbc152b7f" + ], + "index": "pypi", + "version": "==0.17" + }, "sphinxcontrib-qthelp": { "hashes": [ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", @@ -1215,13 +1223,13 @@ }, "typing-extensions": { "hashes": [ - "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", - "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", - "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", + "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", + "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", "python_version <": "3.8", - "version": "==3.7.4.3", + "version": "==3.10.0.0", "version >=": "3.7.4" }, "urllib3": { @@ -1234,11 +1242,11 @@ }, "virtualenv": { "hashes": [ - "sha256:09c61377ef072f43568207dc8e46ddeac6bcdcaf288d49011bda0e7f4d38c4a2", - "sha256:a935126db63128861987a7d5d30e23e8ec045a73840eeccb467c148514e29535" + "sha256:307a555cf21e1550885c82120eccaf5acedf42978fd362d32ba8410f9593f543", + "sha256:72cf267afc04bf9c86ec932329b7e94db6a0331ae9847576daaa7ca3c86b29a4" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4.4" + "version": "==20.4.6" }, "webencodings": { "hashes": [ diff --git a/docs/conf.py b/docs/conf.py index d5673738452..7162f13dd41 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,6 +59,7 @@ def make_pypi_svg(version: str) -> None: "sphinx.ext.intersphinx", "sphinx.ext.napoleon", "myst_parser", + "sphinxcontrib.programoutput", ] # If you need extensions of a certain version or higher, list them here. diff --git a/docs/contributing/gauging_changes.md b/docs/contributing/gauging_changes.md index 6b70e0bc9bc..b41c7a35dda 100644 --- a/docs/contributing/gauging_changes.md +++ b/docs/contributing/gauging_changes.md @@ -37,25 +37,6 @@ If you're running locally yourself to test black on lots of code try: ### CLI arguments -```text -Usage: black-primer [OPTIONS] +```{program-output} black-primer --help - primer - prime projects for blackening... 🏴 - -Options: - -c, --config PATH JSON config file path [default: /Users/cooper/repos/ - black/src/black_primer/primer.json] - - --debug Turn on debug logging [default: False] - -k, --keep Keep workdir + repos post run [default: False] - -L, --long-checkouts Pull big projects to test [default: False] - -R, --rebase Rebase project if already checked out [default: - False] - - -w, --workdir PATH Directory path for repo checkouts [default: /var/fol - ders/tc/hbwxh76j1hn6gqjd2n2sjn4j9k1glp/T/primer.20200 - 517125229] - - -W, --workers INTEGER Number of parallel worker coroutines [default: 2] - -h, --help Show this message and exit. ``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 6c34e1fefab..c65cbe3c476 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ -MyST-Parser==0.13.7 +MyST-Parser==0.14.0 Sphinx==3.5.4 +sphinxcontrib-programoutput==0.17 diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index 0c0382b3e50..75a4d925a54 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -18,14 +18,8 @@ formatting requests. `blackd` provides even less options than _Black_. You can see them by running `blackd --help`: -```text -Usage: blackd [OPTIONS] - -Options: - --bind-host TEXT Address to bind the server to. - --bind-port INTEGER Port to listen on - --version Show the version and exit. - -h, --help Show this message and exit. +```{program-output} blackd --help + ``` There is no official `blackd` client tool (yet!). You can test that blackd is working diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 6fd8769b591..4ac1693ea8b 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -34,92 +34,8 @@ running `black --help`. Help output -``` - Usage: black [OPTIONS] [SRC]... - - The uncompromising code formatter. - - Options: - -c, --code TEXT Format the code passed in as a string. - -l, --line-length INTEGER How many characters per line to allow. - [default: 88] - - -t, --target-version [py27|py33|py34|py35|py36|py37|py38|py39] - Python versions that should be supported by - Black's output. [default: per-file auto- - detection] - - --pyi Format all input files like typing stubs - regardless of file extension (useful when - piping source on standard input). - - -S, --skip-string-normalization - Don't normalize string quotes or prefixes. - -C, --skip-magic-trailing-comma - Don't use trailing commas as a reason to - split lines. - - --check Don't write the files back, just return the - status. Return code 0 means nothing would - change. Return code 1 means some files - would be reformatted. Return code 123 means - there was an internal error. - - --diff Don't write the files back, just output a - diff for each file on stdout. - - --color / --no-color Show colored diff. Only applies when - `--diff` is given. - - --fast / --safe If --fast given, skip temporary sanity - checks. [default: --safe] - - --include TEXT A regular expression that matches files and - directories that should be included on - recursive searches. An empty value means - all files are included regardless of the - name. Use forward slashes for directories - on all platforms (Windows, too). Exclusions - are calculated first, inclusions later. - [default: \.pyi?$] - - --exclude TEXT A regular expression that matches files and - directories that should be excluded on - recursive searches. An empty value means no - paths are excluded. Use forward slashes for - directories on all platforms (Windows, too). - Exclusions are calculated first, inclusions - later. [default: /(\.direnv|\.eggs|\.git|\. - hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_bu - ild|buck-out|build|dist)/] - - --extend-exclude TEXT Like --exclude, but adds additional files - and directories on top of the excluded - ones (useful if you simply want to add to - the default). - - --force-exclude TEXT Like --exclude, but files and directories - matching this regex will be excluded even - when they are passed explicitly as - arguments. - - - --stdin-filename TEXT The name of the file when passing it through - stdin. Useful to make sure Black will - respect --force-exclude option on some - editors that rely on using stdin. - - -q, --quiet Don't emit non-error messages to stderr. - Errors are still emitted; silence those with - 2>/dev/null. - - -v, --verbose Also emit messages to stderr about files - that were not changed or were ignored due to - exclusion patterns. - - --version Show the version and exit. - --config FILE Read configuration from FILE path. - -h, --help Show this message and exit. +```{program-output} black --help + ```
From 7c851dfa2c2a8f0bf4c529d7f6faa43798d04325 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Mon, 10 May 2021 10:57:22 -0400 Subject: [PATCH 66/81] Cover more in the usage docs (#2208) Commit history before merge: * Cover more in the usage docs * Minor fixes * Even more corrections by Jelle * Update docs/usage_and_configuration/the_basics.md Co-authored-by: Jelle Zijlstra --- docs/usage_and_configuration/the_basics.md | 143 +++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 4ac1693ea8b..0b2cd3b3544 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -40,6 +40,149 @@ running `black --help`. +### Code input alternatives + +#### Standard Input + +_Black_ supports formatting code via stdin, with the result being printed to stdout. +Just let _Black_ know with `-` as the path. + +```console +$ echo "print ( 'hello, world' )" | black - +print("hello, world") +reformatted - +All done! ✨ 🍰 ✨ +1 file reformatted. +``` + +**Tip:** if you need _Black_ to treat stdin input as a file passed directly via the CLI, +use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` +option on some editors that rely on using stdin. + +#### As a string + +You can also pass code as a string using the `-c` / `--code` option. + +```console +$ black --code "print ( 'hello, world' )" +print("hello, world") +``` + +```{warning} +--check, --diff, and --safe / --fast have no effect when using -c / --code. Safety +checks normally turned on by default that verify _Black_'s output are disabled as well. +This is a bug which we intend to fix eventually. More details can be found in this [bug +report](https://github.com/psf/black/issues/2104). +``` + +### Writeback and reporting + +By default _Black_ reformats the files given and/or found in place. Sometimes you need +_Black_ to just tell you what it _would_ do without actually rewriting the Python files. + +There's two variations to this mode that are independently enabled by their respective +flags. Both variations can be enabled at once. + +#### Exit code + +Passing `--check` will make _Black_ exit with: + +- code 0 if nothing would change; +- code 1 if some files would be reformatted; or +- code 123 if there was an internal error + +```console +$ black test.py --check +All done! ✨ 🍰 ✨ +1 file would be left unchanged. +$ echo $? +0 + +$ black test.py --check +would reformat test.py +Oh no! 💥 💔 💥 +1 file would be reformatted. +$ echo $? +1 + +$ black test.py --check +error: cannot format test.py: INTERNAL ERROR: Black produced code that is not equivalent to the source. Please report a bug on https://github.com/psf/black/issues. This diff might be helpful: /tmp/blk_kjdr1oog.log +Oh no! 💥 💔 💥 +1 file would fail to reformat. +$ echo $? +123 +``` + +#### Diffs + +Passing `--diff` will make _Black_ print out diffs that indicate what changes _Black_ +would've made. They are printed to stdout so capturing them is simple. + +If you'd like colored diffs, you can enable them with the `--color`. + +```console +$ black test.py --diff +--- test.py 2021-03-08 22:23:40.848954 +0000 ++++ test.py 2021-03-08 22:23:47.126319 +0000 +@@ -1 +1 @@ +-print ( 'hello, world' ) ++print("hello, world") +would reformat test.py +All done! ✨ 🍰 ✨ +1 file would be reformatted. +``` + +### Output verbosity + +_Black_ in general tries to produce the right amount of output, balancing between +usefulness and conciseness. By default, _Black_ emits files modified and error messages, +plus a short summary. + +```console +$ black src/ +error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +reformatted src/black_primer/lib.py +reformatted src/blackd/__init__.py +reformatted src/black/__init__.py +Oh no! 💥 💔 💥 +3 files reformatted, 2 files left unchanged, 1 file failed to reformat. +``` + +Passing `-v` / `--verbose` will cause _Black_ to also emit messages about files that +were not changed or were ignored due to exclusion patterns. If _Black_ is using a +configuration file, a blue message detailing which one it is using will be emitted. + +```console +$ black src/ -v +Using configuration from /tmp/pyproject.toml. +src/blib2to3 ignored: matches the --extend-exclude regular expression +src/_black_version.py wasn't modified on disk since last run. +src/black/__main__.py wasn't modified on disk since last run. +error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +reformatted src/black_primer/lib.py +reformatted src/blackd/__init__.py +reformatted src/black/__init__.py +Oh no! 💥 💔 💥 +3 files reformatted, 2 files left unchanged, 1 file failed to reformat +``` + +Passing `-q` / `--quiet` will cause _Black_ to stop emitting all non-critial output. +Error messages will still be emitted (which can silenced by `2>/dev/null`). + +```console +$ black src/ -q +error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +``` + +### Getting the version + +You can check the version of _Black_ you have installed using the `--version` flag. + +```console +$ black --version +black, version 21.5b0 +``` + ## Configuration via a file _Black_ is able to read project-specific default values for its command line options From f1ce47bd2b0150d2b7c78beaafd3b60c469345d0 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 10 May 2021 07:58:36 -0700 Subject: [PATCH 67/81] Release process docs (#2214) * Setup groundwork for release process docs I'm using MyST for the index page since I like it more and it's easier to work with. * Fill in Release Process for black * Apply suggestions from code review Apply Jelle's grammar + typo fixes. I am a terrible only English speaker. Co-authored-by: Jelle Zijlstra * Update release_process.md Make lint happy via web UI. * Move to contribution section and fix prettier Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- docs/contributing/index.rst | 2 + docs/contributing/release_process.md | 72 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 docs/contributing/release_process.md diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 68dfcd60c28..e7a7f9fb231 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -6,6 +6,7 @@ Contributing the_basics gauging_changes + release_process reference/reference_summary Welcome! Happy to see you willing to make the project better. Have you read the entire @@ -31,6 +32,7 @@ This section covers the following topics: - :doc:`the_basics` - :doc:`gauging_changes` +- :doc:`release_process` - :doc:`reference/reference_summary` For an overview on contributing to the *Black*, please checkout :doc:`the_basics`. diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md new file mode 100644 index 00000000000..fadef4bbe3d --- /dev/null +++ b/docs/contributing/release_process.md @@ -0,0 +1,72 @@ +# Release process + +_Black_ has had a lot of work automating its release process. This document sets out to +explain what everything does and how to release _Black_ using said automation. + +## Cutting a Relase + +To cut a realease, you must be a _Black_ maintainer with `GitHub Release` creation +access. Using this access, the release process is: + +1. Cut a new PR editing `CHANGES.md` to version the latest changes + 1. Example PR: https://github.com/psf/black/pull/2192 + 2. Example title: `Update CHANGES.md for XX.X release` +2. Once the release PR is merged ensure all CI passes + 1. If not, ensure there is an Issue open for the cause of failing CI (generally we'd + want this fixed before cutting a release) +3. Open `CHANGES.md` and copy the _raw markdown_ of the latest changes to use in the + description of the GitHub Release. +4. Go and [cut a release](https://github.com/psf/black/releases) using the GitHub UI so + that all workflows noted below are triggered. + 1. The release version and tag should be the [CalVer](https://calver.org) version + _Black_ used for the current release e.g. `21.6` / `21.5b1` + 2. _Black_ uses [setuptools scm](https://pypi.org/project/setuptools-scm/) to pull + the current version for the package builds and release. +5. Once the release is cut, you're basically done. It's a good practice to go and watch + to make sure all the [GitHub Actions](https://github.com/psf/black/actions) pass, + although you should receive an email to your registered GitHub email address should + one fail. + 1. You should see all the release workflows and lint/unittests workflows running on + the new tag in the Actions UI + +If anything fails, please go read the respective action's log output and configuration +file to reverse engineer your way to a fix/soluton. + +## Release workflows + +All _Blacks_'s automation workflows use GitHub Actions. All workflows are therefore +configured using `.yml` files in the `.github/workflows` directory of the _Black_ +repository. + +Below are descriptions of our release workflows. + +### Docker + +This workflow uses the QEMU powered `buildx` feature of docker to upload a `arm64` and +`amd64`/`x86_64` build of the official _Black_ docker image™. + +- Currently this workflow uses an API Token associated with @cooperlees account + +### pypi_upload + +This workflow builds a Python +[sdist](https://docs.python.org/3/distutils/sourcedist.html) and +[wheel](https://pythonwheels.com) using the latest +[setuptools](https://pypi.org/project/setuptools/) and +[wheel](https://pypi.org/project/wheel/) modules. + +It will then use [twine](https://pypi.org/project/twine/) to upload both release formats +to PyPI for general downloading of the _Black_ Python package. This is where +[pip](https://pypi.org/project/pip/) looks by default. + +- Currently this workflow uses an API token associated with @ambv's PyPI account + +### Upload self-contained binaries + +This workflow builds self-contained binaries for multiple platforms. This allows people +to download the executable for their platform and run _Black_ without a +[Python Runtime](https://wiki.python.org/moin/PythonImplementations) installed. + +The created binaries are attached/stored on the associated +[GitHub Release](https://github.com/psf/black/releases) for download over _IPv4 only_ +(GitHub still does not have IPv6 access 😢). From f933e8a8382e6945bf950356f6ecc4ded5617617 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 10 May 2021 07:59:32 -0700 Subject: [PATCH 68/81] Update CHANGES.md for 21.5b1 release (#2215) --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 32c2d8d7237..38eef524b7c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Change Log -## Unreleased +## 21.5b1 ### _Black_ From 2f52e4b4929370ec503ee272bcc10d3176db8e89 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 10 May 2021 08:01:53 -0700 Subject: [PATCH 69/81] fix typo (#2217) --- docs/contributing/release_process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index fadef4bbe3d..ae95cd703c9 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -5,7 +5,7 @@ explain what everything does and how to release _Black_ using said automation. ## Cutting a Relase -To cut a realease, you must be a _Black_ maintainer with `GitHub Release` creation +To cut a release, you must be a _Black_ maintainer with `GitHub Release` creation access. Using this access, the release process is: 1. Cut a new PR editing `CHANGES.md` to version the latest changes From 53d9bace12b3aa230820c869a079020b4608c945 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 11 May 2021 10:01:03 -0700 Subject: [PATCH 70/81] Add stable tag process to release process documentation (#2224) * Add stable tag process to release process documentation - Add reasoning + step commands * Bah - I ran the linter but forgot to commit * Update docs/contributing/release_process.md Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- docs/contributing/release_process.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index ae95cd703c9..718ea3dc9a2 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -3,7 +3,7 @@ _Black_ has had a lot of work automating its release process. This document sets out to explain what everything does and how to release _Black_ using said automation. -## Cutting a Relase +## Cutting a Release To cut a release, you must be a _Black_ maintainer with `GitHub Release` creation access. Using this access, the release process is: @@ -70,3 +70,20 @@ to download the executable for their platform and run _Black_ without a The created binaries are attached/stored on the associated [GitHub Release](https://github.com/psf/black/releases) for download over _IPv4 only_ (GitHub still does not have IPv6 access 😢). + +## Moving the `stable` tag + +_Black_ provides a stable tag for people who want to move along as _Black_ developers +deem the newest version reliable. Here the _Black_ developers will move once the release +has been problem free for at least ~24 hours from release. Given the large _Black_ +userbase we hear about bad bugs quickly. We do strive to continually improve our CI too. + +### Tag moving process + +#### stable + +From a rebased `main` checkout: + +1. `git tag -f stable VERSION_TAG` + 1. e.g. `git tag -f stable 21.5b1` +1. `git push --tags -f` From 94a0b07dbebabe344991dc27f9eb0b11597bb3b5 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 11 May 2021 14:09:33 -0400 Subject: [PATCH 71/81] Remove useless flake8 config + test support code (#2221) We've depended on Click 7.x ever since we broke CI systems across the world (oops lol) and flake8-mypy was purged a fair bit back: #1867 Also remove the primer tests import in tests/test_black.py because it's annoying when just trying to actually target tests/test_black.py tests. `pytest -k test_black.py` doesn't do what you expect due to that import. --- .flake8 | 4 ---- tests/test_black.py | 35 +++++++---------------------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/.flake8 b/.flake8 index 68d15015a37..4866133a6a8 100644 --- a/.flake8 +++ b/.flake8 @@ -5,7 +5,3 @@ extend-ignore = E203, E266, E501 max-line-length = 80 max-complexity = 18 select = B,C,E,F,W,T4,B9 -# We need to configure the mypy.ini because the flake8-mypy's default -# options don't properly override it, so if we don't specify it we get -# half of the config from mypy.ini and half from flake8-mypy. -mypy_config = mypy.ini diff --git a/tests/test_black.py b/tests/test_black.py index 347eaf621fe..4cbb022954f 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from dataclasses import replace import inspect -from io import BytesIO, TextIOWrapper +from io import BytesIO import os from pathlib import Path from platform import system @@ -16,10 +16,8 @@ import types from typing import ( Any, - BinaryIO, Callable, Dict, - Generator, List, Iterator, TypeVar, @@ -52,7 +50,6 @@ ff, dump_to_stderr, ) -from .test_primer import PrimerCLITests # noqa: F401 THIS_FILE = Path(__file__) @@ -104,28 +101,10 @@ def __init__(self) -> None: class BlackRunner(CliRunner): - """Modify CliRunner so that stderr is not merged with stdout. - - This is a hack that can be removed once we depend on Click 7.x""" + """Make sure STDOUT and STDERR are kept seperate when testing Black via its CLI.""" def __init__(self) -> None: - self.stderrbuf = BytesIO() - self.stdoutbuf = BytesIO() - self.stdout_bytes = b"" - self.stderr_bytes = b"" - super().__init__() - - @contextmanager - def isolation(self, *args: Any, **kwargs: Any) -> Generator[BinaryIO, None, None]: - with super().isolation(*args, **kwargs) as output: - try: - hold_stderr = sys.stderr - sys.stderr = TextIOWrapper(self.stderrbuf, encoding=self.charset) - yield output - finally: - self.stdout_bytes = sys.stdout.buffer.getvalue() # type: ignore - self.stderr_bytes = sys.stderr.buffer.getvalue() # type: ignore - sys.stderr = hold_stderr + super().__init__(mix_stderr=False) class BlackTestCase(BlackBaseTestCase): @@ -141,8 +120,8 @@ def invokeBlack( exit_code, msg=( f"Failed with args: {args}\n" - f"stdout: {runner.stdout_bytes.decode()!r}\n" - f"stderr: {runner.stderr_bytes.decode()!r}\n" + f"stdout: {result.stdout_bytes.decode()!r}\n" + f"stderr: {result.stderr_bytes.decode()!r}\n" f"exception: {result.exception}" ), ) @@ -483,7 +462,7 @@ def test_python2_should_fail_without_optional_install(self) -> None: finally: os.unlink(tmp_file) actual = ( - runner.stderr_bytes.decode() + result.stderr_bytes.decode() .replace("\n", "") .replace("\\n", "") .replace("\\r", "") @@ -1806,7 +1785,7 @@ def test_preserves_line_endings_via_stdin(self) -> None: black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf8")) ) self.assertEqual(result.exit_code, 0) - output = runner.stdout_bytes + output = result.stdout_bytes self.assertIn(nl.encode("utf8"), output) if nl == "\n": self.assertNotIn(b"\r\n", output) From b2ee211b5ad84b62738ac0997b73bf6ee9a74d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 12 May 2021 21:47:32 +0200 Subject: [PATCH 72/81] Click 8.0 renamed its "die on LANG=C" function so we need to look for that one too (#2227) --- CHANGES.md | 6 ++++++ mypy.ini | 6 ++++-- src/black/__init__.py | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 38eef524b7c..becd621dbf2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Change Log +## Unreleased + +### _Black_ + +- Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) + ## 21.5b1 ### _Black_ diff --git a/mypy.ini b/mypy.ini index 589fbf65139..ae7be5e5106 100644 --- a/mypy.ini +++ b/mypy.ini @@ -33,5 +33,7 @@ cache_dir=/dev/null [mypy-aiohttp.*] follow_imports=skip -[mypy-_version] -follow_imports=skip +[mypy-black] +# The following is because of `patch_click()`. Remove when +# we drop Python 3.6 support. +warn_unused_ignores=False diff --git a/src/black/__init__.py b/src/black/__init__.py index c61bc8c1d60..f46b866c1bd 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1029,7 +1029,7 @@ def nullcontext() -> Iterator[None]: def patch_click() -> None: - """Make Click not crash. + """Make Click not crash on Python 3.6 with LANG=C. On certain misconfigured environments, Python 3 selects the ASCII encoding as the default which restricts paths that it can access during the lifetime of the @@ -1047,7 +1047,9 @@ def patch_click() -> None: for module in (core, _unicodefun): if hasattr(module, "_verify_python3_env"): - module._verify_python3_env = lambda: None + module._verify_python3_env = lambda: None # type: ignore + if hasattr(module, "_verify_python_env"): + module._verify_python_env = lambda: None # type: ignore def patched_main() -> None: From 3ef339b2e75468a09d617e6aa74bc920c317bce6 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Wed, 12 May 2021 21:28:41 -0400 Subject: [PATCH 73/81] Modify when Test, Primer, and Documentation Build run (#2226) - Test and Primer don't run for documentation only changes since it's unnecessary, eating unnecessary cycles and slowing down CI since these workflows eat up the 20 max workers limit quite easily! - Documentation Build runs all of the time now since quite a bit of the content depends on Black's code so even a simple 1-file change in src/black/__init__.py may break the docs build. It's not like this is a costly workflow anyway. Fuzz is still running on all changes because with fuzzing, the more the better in general. 6 or 7 jobs on a documentation only commit is much better than 27/28 jobs anyway :p I also found an error in our bug report issue template :) --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/workflows/doc.yml | 16 +--------------- .github/workflows/primer.yml | 11 ++++++++++- .github/workflows/test.yml | 11 ++++++++++- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index c6c80be5345..069795f7776 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -30,7 +30,7 @@ assignees: "" - clone this repository; - run `pip install -e .[d,python2]`; - run `pip install -r test_requirements.txt` - - make sure it's sane by running `python -m unittest`; and + - make sure it's sane by running `python -m pytest`; and - run `black` like you did last time. **Additional context** Add any other context about the problem here. diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 74ec316a7cf..04b25cf2a16 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -1,20 +1,6 @@ name: Documentation Build -on: - push: - paths: - - "docs/**" - - "README.md" - - "CHANGES.md" - - "CONTRIBUTING.md" - - "AUTHORS.md" - pull_request: - paths: - - "docs/**" - - "README.md" - - "CHANGES.md" - - "CONTRIBUTING.md" - - "AUTHORS.md" +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/primer.yml b/.github/workflows/primer.yml index 4c5751ae996..5f41c301737 100644 --- a/.github/workflows/primer.yml +++ b/.github/workflows/primer.yml @@ -1,6 +1,15 @@ name: Primer -on: [push, pull_request] +on: + push: + paths-ignore: + - "docs/**" + - "*.md" + + pull_request: + paths-ignore: + - "docs/**" + - "*.md" jobs: build: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2cfbab67ce1..ef8debb3fb7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,15 @@ name: Test -on: [push, pull_request] +on: + push: + paths-ignore: + - "docs/**" + - "*.md" + + pull_request: + paths-ignore: + - "docs/**" + - "*.md" jobs: build: From 445f094f1fa2d998bf0cc0007ea48d62953fa876 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 13 May 2021 19:28:41 +0200 Subject: [PATCH 74/81] Use codespell to find typos (#2228) --- CHANGES.md | 1 + docs/index.rst | 2 +- src/black/trans.py | 2 +- src/black_primer/lib.py | 2 +- tests/test_black.py | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index becd621dbf2..3639d8ebaf9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ ### _Black_ +- Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) ## 21.5b1 diff --git a/docs/index.rst b/docs/index.rst index a7a7160f71f..2b85cddd3c0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,7 @@ Try it out now using the `Black Playground `_. many projects, small and big. *Black* has a comprehensive test suite, with efficient parallel tests, our own auto formatting and parallel Continuous Integration runner. However, *Black* is still beta. Things will probably be wonky for a while. This is - made explicit by the "Beta" trove classifier, as well as by the "b" in the versio + made explicit by the "Beta" trove classifier, as well as by the "b" in the version number. What this means for you is that **until the formatter becomes stable, you should expect some formatting to change in the future**. That being said, no drastic stylistic changes are planned, mostly responses to bug reports. diff --git a/src/black/trans.py b/src/black/trans.py index 055f33c0ee2..7ecc31d6d31 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -576,7 +576,7 @@ class StringParenStripper(StringTransformer): - The target string is NOT the only argument to a function call. - The target string is NOT a "pointless" string. - If the target string contains a PERCENT, the brackets are not - preceeded or followed by an operator with higher precedence than + preceded or followed by an operator with higher precedence than PERCENT. Transformations: diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index 3ce383f17ce..384c0ad6cea 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -128,7 +128,7 @@ async def black_run( cmd.extend(["--diff", "."]) with TemporaryDirectory() as tmp_path: - # Prevent reading top-level user configs by manipulating envionment variables + # Prevent reading top-level user configs by manipulating environment variables env = { **os.environ, "XDG_CONFIG_HOME": tmp_path, # Unix-like diff --git a/tests/test_black.py b/tests/test_black.py index 4cbb022954f..5ab25cd1601 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -101,7 +101,7 @@ def __init__(self) -> None: class BlackRunner(CliRunner): - """Make sure STDOUT and STDERR are kept seperate when testing Black via its CLI.""" + """Make sure STDOUT and STDERR are kept separate when testing Black via its CLI.""" def __init__(self) -> None: super().__init__(mix_stderr=False) @@ -1508,7 +1508,7 @@ def test_get_sources_with_stdin_filename(self) -> None: @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) def test_get_sources_with_stdin_filename_and_exclude(self) -> None: - # Exclude shouldn't exclude stdin_filename since it is mimicing the + # Exclude shouldn't exclude stdin_filename since it is mimicking the # file being passed directly. This is the same as # test_exclude_for_issue_1572 path = THIS_DIR / "data" / "include_exclude_tests" From 904fe94ceba29f3f0c5b8772c4faf699de57338f Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Thu, 13 May 2021 15:30:34 -0400 Subject: [PATCH 75/81] Add lower bound for aiohttp-cors + fix primer (#2231) It appears sqlalchemy has recently reformatted their project with Black 21.5b1. Most of our dependencies have a lower bound and creating a test environment with the oldest acceptable dependencies runs the full Black test suite just fine. The only exception to this is aiohttp-cors. It's unbounded and the oldest version 0.1.0 until 0.4.0 breaks the test suite in such an old environment. Failure with 0.1.0: ``` tests/test_blackd.py:10: in import blackd testenv/lib/python3.8/site-packages/blackd/__init__.py:12: in import aiohttp_cors testenv/lib/python3.8/site-packages/aiohttp_cors/__init__.py:29: in from .urldispatcher_router_adapter import UrlDistatcherRouterAdapter testenv/lib/python3.8/site-packages/aiohttp_cors/urldispatcher_router_adapter.py:27: in class UrlDistatcherRouterAdapter(RouterAdapter): testenv/lib/python3.8/site-packages/aiohttp_cors/urldispatcher_router_adapter.py:32: in UrlDistatcherRouterAdapter def route_methods(self, route: web.Route): E AttributeError: module 'aiohttp.web' has no attribute 'Route' ``` For 0.2.0: ``` tests/test_blackd.py:10: in import blackd testenv/lib/python3.8/site-packages/blackd/__init__.py:12: in import aiohttp_cors testenv/lib/python3.8/site-packages/aiohttp_cors/__init__.py:27: in from .cors_config import CorsConfig testenv/lib/python3.8/site-packages/aiohttp_cors/cors_config.py:24: in from .urldispatcher_router_adapter import UrlDistatcherRouterAdapter testenv/lib/python3.8/site-packages/aiohttp_cors/urldispatcher_router_adapter.py:27: in class UrlDistatcherRouterAdapter(AbstractRouterAdapter): testenv/lib/python3.8/site-packages/aiohttp_cors/urldispatcher_router_adapter.py:32: in UrlDistatcherRouterAdapter def route_methods(self, route: web.Route): E AttributeError: module 'aiohttp.web' has no attribute 'Route' ``` For 0.3.0: ``` ERROR: Cannot install aiohttp-cors==0.3.0 and aiohttp==3.6.0 because these package versions have conflicting dependencies. The conflict is caused by: The user requested aiohttp==3.6.0 aiohttp-cors 0.3.0 depends on aiohttp<=0.20.2 and >=0.18.0 To fix this you could try to: 1. loosen the range of package versions you've specified 2. remove package versions to allow pip attempt to solve the dependency conflict ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/user_guide/#fixing-conflicting-dependencies ``` --- CHANGES.md | 5 +++++ setup.py | 2 +- src/black_primer/primer.json | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3639d8ebaf9..85936e870b1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,11 @@ - Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) +### _Blackd_ + +- Add a lower bound for the `aiohttp-cors` dependency. Only 0.4.0 or higher is + supported. (#2231) + ## 21.5b1 ### _Black_ diff --git a/setup.py b/setup.py index 3dc52b26e81..2b5f71f56dd 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def get_long_description() -> str: "mypy_extensions>=0.4.3", ], extras_require={ - "d": ["aiohttp>=3.6.0", "aiohttp-cors"], + "d": ["aiohttp>=3.6.0", "aiohttp-cors>=0.4.0"], "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], }, diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 76ed4820487..0bbac85ca2e 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -106,7 +106,7 @@ "sqlalchemy": { "no_cli_args_reason": "breaks black with new string parsing - #2188", "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git", "long_checkout": false, "py_versions": ["all"] From 403ce1a18a8a1600ab8249d828e7eaaca442cad7 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sun, 16 May 2021 12:07:27 -0400 Subject: [PATCH 76/81] Add issue triage documentation (#2236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add issue triage documentation Co-authored-by: Łukasz Langa Co-authored-by: Jelle Zijlstra --- Pipfile | 3 +- Pipfile.lock | 120 ++++++++++----------- docs/conf.py | 1 + docs/contributing/index.rst | 1 + docs/contributing/issue_triage.md | 167 ++++++++++++++++++++++++++++++ docs/requirements.txt | 3 + 6 files changed, 229 insertions(+), 66 deletions(-) create mode 100644 docs/contributing/issue_triage.md diff --git a/Pipfile b/Pipfile index b3859571137..789d97c3cbc 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ pre-commit = "*" readme_renderer = "*" MyST-Parser = ">=0.13.7" sphinxcontrib-programoutput = ">=0.17" +sphinx-copybutton = ">=0.3.0" setuptools = ">=39.2.0" setuptools-scm = "*" twine = ">=1.11.0" @@ -22,7 +23,7 @@ black = {editable = true, extras = ["d"], path = "."} [packages] aiohttp = ">=3.6.0" -aiohttp-cors = "*" +aiohttp-cors = ">=0.4.0" appdirs = "*" click = ">=7.1.2" mypy_extensions = ">=0.4.3" diff --git a/Pipfile.lock b/Pipfile.lock index 0b9e529e73b..b6dfe438e9e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fdfbacb362e4514e588736a7e8783837cb5e3aa2fbab98ea17894fdb50d51a8e" + "sha256": "b450e698cfe4d856397cb9d0695dcd74eb97b1d234de273179d78c0d0efccd0f" }, "pipfile-spec": 6, "requires": {}, @@ -106,11 +106,11 @@ }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:7d8c289ee437bcb0316820ccee14aefcb056e58d31830ecab8e47eda6540e136", + "sha256:e90e62ced43dc8105fb9a26d62f0d9340b5c8db053a814e25d95c19873ae87db" ], "index": "pypi", - "version": "==7.1.2" + "version": "==8.0.0" }, "dataclasses": { "hashes": [ @@ -516,11 +516,11 @@ }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:7d8c289ee437bcb0316820ccee14aefcb056e58d31830ecab8e47eda6540e136", + "sha256:e90e62ced43dc8105fb9a26d62f0d9340b5c8db053a814e25d95c19873ae87db" ], "index": "pypi", - "version": "==7.1.2" + "version": "==8.0.0" }, "colorama": { "hashes": [ @@ -687,11 +687,11 @@ }, "jinja2": { "hashes": [ - "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", - "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" + "sha256:2f2de5285cf37f33d33ecd4a9080b75c87cd0c1994d5a9c6df17131ea1f049c6", + "sha256:ea8d7dd814ce9df6de6a761ec7f1cac98afe305b8cdc4aaae4e114b8d8ce24c5" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.11.3" + "markers": "python_version >= '3.6'", + "version": "==3.0.0" }, "keyring": { "hashes": [ @@ -711,61 +711,43 @@ }, "markupsafe": { "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", - "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", - "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", - "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", - "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", - "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", - "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", - "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", - "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" + "sha256:007dc055dbce5b1104876acee177dbfd18757e19d562cd440182e1f492e96b95", + "sha256:031bf79a27d1c42f69c276d6221172417b47cb4b31cdc73d362a9bf5a1889b9f", + "sha256:161d575fa49395860b75da5135162481768b11208490d5a2143ae6785123e77d", + "sha256:24bbc3507fb6dfff663af7900a631f2aca90d5a445f272db5fc84999fa5718bc", + "sha256:2efaeb1baff547063bad2b2893a8f5e9c459c4624e1a96644bbba08910ae34e0", + "sha256:32200f562daaab472921a11cbb63780f1654552ae49518196fc361ed8e12e901", + "sha256:3261fae28155e5c8634dd7710635fe540a05b58f160cef7713c7700cb9980e66", + "sha256:3b54a9c68995ef4164567e2cd1a5e16db5dac30b2a50c39c82db8d4afaf14f63", + "sha256:3c352ff634e289061711608f5e474ec38dbaa21e3e168820d53d5f4015e5b91b", + "sha256:3fb47f97f1d338b943126e90b79cad50d4fcfa0b80637b5a9f468941dbbd9ce5", + "sha256:441ce2a8c17683d97e06447fcbccbdb057cbf587c78eb75ae43ea7858042fe2c", + "sha256:45535241baa0fc0ba2a43961a1ac7562ca3257f46c4c3e9c0de38b722be41bd1", + "sha256:4aca81a687975b35e3e80bcf9aa93fe10cd57fac37bf18b2314c186095f57e05", + "sha256:4cc563836f13c57f1473bc02d1e01fc37bab70ad4ee6be297d58c1d66bc819bf", + "sha256:4fae0677f712ee090721d8b17f412f1cbceefbf0dc180fe91bab3232f38b4527", + "sha256:58bc9fce3e1557d463ef5cee05391a05745fd95ed660f23c1742c711712c0abb", + "sha256:664832fb88b8162268928df233f4b12a144a0c78b01d38b81bdcf0fc96668ecb", + "sha256:70820a1c96311e02449591cbdf5cd1c6a34d5194d5b55094ab725364375c9eb2", + "sha256:79b2ae94fa991be023832e6bcc00f41dbc8e5fe9d997a02db965831402551730", + "sha256:83cf0228b2f694dcdba1374d5312f2277269d798e65f40344964f642935feac1", + "sha256:87de598edfa2230ff274c4de7fcf24c73ffd96208c8e1912d5d0fee459767d75", + "sha256:8f806bfd0f218477d7c46a11d3e52dc7f5fdfaa981b18202b7dc84bbc287463b", + "sha256:90053234a6479738fd40d155268af631c7fca33365f964f2208867da1349294b", + "sha256:a00dce2d96587651ef4fa192c17e039e8cfab63087c67e7d263a5533c7dad715", + "sha256:a08cd07d3c3c17cd33d9e66ea9dee8f8fc1c48e2d11bd88fd2dc515a602c709b", + "sha256:a19d39b02a24d3082856a5b06490b714a9d4179321225bbf22809ff1e1887cc8", + "sha256:d00a669e4a5bec3ee6dbeeeedd82a405ced19f8aeefb109a012ea88a45afff96", + "sha256:dab0c685f21f4a6c95bfc2afd1e7eae0033b403dd3d8c1b6d13a652ada75b348", + "sha256:df561f65049ed3556e5b52541669310e88713fdae2934845ec3606f283337958", + "sha256:e4570d16f88c7f3032ed909dc9e905a17da14a1c4cfd92608e3fda4cb1208bbd", + "sha256:e77e4b983e2441aff0c0d07ee711110c106b625f440292dfe02a2f60c8218bd6", + "sha256:e79212d09fc0e224d20b43ad44bb0a0a3416d1e04cf6b45fed265114a5d43d20", + "sha256:f58b5ba13a5689ca8317b98439fccfbcc673acaaf8241c1869ceea40f5d585bf", + "sha256:fef86115fdad7ae774720d7103aa776144cf9b66673b4afa9bcaa7af990ed07b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.1" + "markers": "python_version >= '3.6'", + "version": "==2.0.0" }, "mccabe": { "hashes": [ @@ -1105,6 +1087,14 @@ "index": "pypi", "version": "==3.5.4" }, + "sphinx-copybutton": { + "hashes": [ + "sha256:0e0461df394515284e3907e3f418a0c60ef6ab6c9a27a800c8552772d0a402a2", + "sha256:5125c718e763596e6e52d92e15ee0d6f4800ad3817939be6dee51218870b3e3d" + ], + "index": "pypi", + "version": "==0.3.1" + }, "sphinxcontrib-applehelp": { "hashes": [ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", diff --git a/docs/conf.py b/docs/conf.py index 7162f13dd41..15adb5df7a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,6 +60,7 @@ def make_pypi_svg(version: str) -> None: "sphinx.ext.napoleon", "myst_parser", "sphinxcontrib.programoutput", + "sphinx_copybutton", ] # If you need extensions of a certain version or higher, list them here. diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index e7a7f9fb231..7e3a90905cd 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -6,6 +6,7 @@ Contributing the_basics gauging_changes + issue_triage release_process reference/reference_summary diff --git a/docs/contributing/issue_triage.md b/docs/contributing/issue_triage.md new file mode 100644 index 00000000000..780b2b5e959 --- /dev/null +++ b/docs/contributing/issue_triage.md @@ -0,0 +1,167 @@ +# Issue triage + +Currently, _Black_ uses the issue tracker for bugs, feature requests, proposed design +modifications, and general user support. Each of these issues have to be triaged so they +can be eventually be resolved somehow. This document outlines the triaging process and +also the current guidelines and recommendations. + +```{tip} +If you're looking for a way to contribute without submitting patches, this might be +the area for you. Since _Black_ is a popular project, its issue tracker is quite busy +and always needs more attention than is available. While triage isn't the most +glamorous or technically challenging form of contribution, it's still important. +For example, we would love to know whether that old bug report is still reproducible! + +You can get easily started by reading over this document and then responding to issues. + +If you contribute enough and have stayed for a long enough time, you may even be +given Triage permissions! +``` + +## The basics + +_Black_ gets a whole bunch of different issues, they range from bug reports to user +support issues. To triage is to identify, organize, and kickstart the issue's journey +through its lifecycle to resolution. + +More specifically, to triage an issue means to: + +- identify what type and categories the issue falls under +- confirm bugs +- ask questions / for further information if necessary +- link related issues +- provide the first initial feedback / support + +Note that triage is typically the first response to an issue, so don't fret if the issue +doesn't make much progress after initial triage. The main goal of triaging to prepare +the issue for future more specific development or discussion, so _eventually_ it will be +resolved. + +The lifecycle of a bug report or user support issue typically goes something like this: + +1. _the issue is waiting for triage_ +2. **identified** - has been marked with a type label and other relevant labels, more + details or a functional reproduction may be still needed (and therefore should be + marked with `S: needs repro` or `S: awaiting reponse`) +3. **confirmed** - the issue can reproduced and necessary details have been provided +4. **discussion** - initial triage has been done and now the general details on how the + issue should be best resolved are being hashed out +5. **awaiting fix** - no further discussion on the issue is necessary and a resolving PR + is the next step +6. **closed** - the issue has been resolved, reasons include: + - the issue couldn't be reproduced + - the issue has been fixed + - duplicate of another pre-existing issue or is invalid + +For enhancement, documentation, and design issues, the lifecycle looks very similar but +the details are different: + +1. _the issue is waiting for triage_ +2. **identified** - has been marked with a type label and other relevant labels +3. **discussion** - the merits of the suggested changes are currently being discussed, a + PR would be acceptable but would be at sigificant risk of being rejected +4. **accepted & awaiting PR** - it's been determined the suggested changes are OK and a + PR would be welcomed (`S: accepted`) +5. **closed**: - the issue has been resolved, reasons include: + - the suggested changes were implemented + - it was rejected (due to technical concerns, ethos conflicts, etc.) + - duplicate of a pre-existing issue or is invalid + +**Note**: documentation issues don't use the `S: accepted` label currently since they're +less likely to be rejected. + +## Labelling + +We use labels to organize, track progress, and help effectively divvy up work. + +Our labels are divided up into several groups identified by their prefix: + +- **T - Type**: the general flavor of issue / PR +- **C - Category**: areas of concerns, ranges from bug types to project maintenance +- **F - Formatting Area**: like C but for formatting specifically +- **S - Status**: what stage of resolution is this issue currently in? +- **R - Resolution**: how / why was the issue / PR resolved? + +We also have a few standalone labels: + +- **`good first issue`**: issues that are beginner-friendly (and will show up in GitHub + banners for first-time visitors to the repository) +- **`help wanted`**: complex issues that need and are looking for a fair bit of work as + to progress (will also show up in various GitHub pages) +- **`skip news`**: for PRs that are trivial and don't need a CHANGELOG entry (and skips + the CHANGELOG entry check) + +```{note} +We do use labels for PRs, in particular the `skip news` label, but we aren't that +rigorous about it. Just follow your judgement on what labels make sense for the +specific PR (if any even make sense). +``` + +## Projects + +For more general and broad goals we use projects to track work. Some may be longterm +projects with no true end (e.g. the "Amazing documentation" project) while others may be +more focused and have a definite end (like the "Getting to beta" project). + +```{note} +To modify GitHub Projects you need the [Write repository permission level or higher](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization#repository-access-for-each-permission-level). +``` + +## Closing issues + +Closing an issue signifies the issue has reached the end of its life, so closing issues +should be taken with care. The following is the general recommendation for each type of +issue. Note that these are only guidelines and if your judgement says something else +it's totally cool to go with it instead. + +For most issues, closing the issue manually or automatically after a resolving PR is +ideal. For bug reports specifically, if the bug has already been fixed, try to check in +with the issue opener that their specific case has been resolved before closing. + +Design and enhancement issues should be also closed when it's clear the proposed change +won't be implemented, whether that has been determined after a lot of discussion or just +simply goes against _Black_'s ethos. If such an issue turns heated, closing and locking +is acceptable if it's severe enough (although checking in with the core team is probably +a good idea). + +User support issues are best closed by the author or when it's clear the issue has been +resolved in some sort of manner. + +Duplicates and invalid issues should always be closed since they serve no purpose and +add noise to an already busy issue tracker. Although be careful to make sure it's truly +a duplicate and not just very similar before labelling and closing an issue as +duplicate. + +## Common reports + +Some issues are frequently opened, like issues about _Black_ formatted code causing E203 +messages. Even though these issues are probably heavily duplicated, they still require +triage sucking up valuable time from other things (although they usually skip most of +their lifecycle since they're closed on triage). + +Here's some of the most common issues and also pre-made responses you can use: + +### "The trailing comma isn't being removed by Black!" + +```text +Black used to remove the trailing comma if the expression fits in a single line, but this was changed by #826 and #1288. Now a trailing comma tells Black to always explode the expression. This change was made mostly for the cases where you _know_ a collection or whatever will grow in the future. Having it always exploded as one element per line reduces diff noise when adding elements. Before the "magic trailing comma" feature, you couldn't anticipate a collection's growth reliably since collections that fitted in one line were ruthlessly collapsed regardless of your intentions. One of Black's goals is reducing diff noise, so this was a good pragmatic change. + +So no, this is not a bug, but an intended feature. Anyway, [here's the documentation](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#the-magic-trailing-comma) on the "magic trailing comma". Hopefully that helps and sorry for the possible confusion. +``` + +### "Black formatted code is violating Flake8's E203!" + +```text +Hi, + +This is expected behaviour, please see the documentation regarding this case (emphasis +mine): + +> PEP 8 recommends to treat : in slices as a binary operator with the lowest priority, and to leave an equal amount of space on either side, **except if a parameter is omitted (e.g. ham[1 + 1 :])**. It recommends no spaces around : operators for “simple expressions” (ham[lower:upper]), and **extra space for “complex expressions” (ham[lower : upper + offset])**. **Black treats anything more than variable names as “complex” (ham[lower : upper + 1]).** It also states that for extended slices, both : operators have to have the same amount of spacing, except if a parameter is omitted (ham[1 + 1 ::]). Black enforces these rules consistently. + +> This behaviour may raise E203 whitespace before ':' warnings in style guide enforcement tools like Flake8. **Since E203 is not PEP 8 compliant, you should tell Flake8 to ignore these warnings**. + +https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices + +Have a good day! +``` diff --git a/docs/requirements.txt b/docs/requirements.txt index c65cbe3c476..56dbf3ffe13 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ +# Used by ReadTheDocs; pinned requirements for stability. + MyST-Parser==0.14.0 Sphinx==3.5.4 sphinxcontrib-programoutput==0.17 +sphinx_copybutton==0.3.1 From 9704922cf92c8f4b26a5efc6ddfdbd42064eae5f Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 16 May 2021 09:10:59 -0700 Subject: [PATCH 77/81] Update vim plugin manual installation instructions. (#2235) --- CHANGES.md | 1 + docs/integrations/editors.md | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 85936e870b1..1a38bf6649b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) +- Fix vim plugin installation instructions. (#2235) ### _Blackd_ diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 88176e1aecb..dd09b4df6d1 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -146,12 +146,15 @@ $ cd ~/.vim/bundle/black $ git checkout origin/stable -b stable ``` -or you can copy the plugin from -[plugin/black.vim](https://github.com/psf/black/blob/stable/plugin/black.vim). +or you can copy the plugin files from +[plugin/black.vim](https://github.com/psf/black/blob/stable/plugin/black.vim) and +[autoload/black.vim](https://github.com/psf/black/blob/stable/autoload/black.vim). ``` mkdir -p ~/.vim/pack/python/start/black/plugin +mkdir -p ~/.vim/pack/python/start/black/autoload curl https://raw.githubusercontent.com/psf/black/stable/plugin/black.vim -o ~/.vim/pack/python/start/black/plugin/black.vim +curl https://raw.githubusercontent.com/psf/black/stable/autoload/black.vim -o ~/.vim/pack/python/start/black/autoload/black.vim ``` Let me know if this requires any changes to work with Vim 8's builtin `packadd`, or From 60f8bd2c89fbb370e6816fd6d6e5eca64d5b9f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 16 May 2021 18:24:28 +0200 Subject: [PATCH 78/81] Include Jelle's review suggestions --- CHANGES.md | 7 +++++-- docs/contributing/issue_triage.md | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1a38bf6649b..f2c795440f8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,15 +4,18 @@ ### _Black_ -- Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) -- Fix vim plugin installation instructions. (#2235) ### _Blackd_ - Add a lower bound for the `aiohttp-cors` dependency. Only 0.4.0 or higher is supported. (#2231) +### Documentation + +- Fix typos discovered by codespell (#2228) +- Fix Vim plugin installation instructions. (#2235) + ## 21.5b1 ### _Black_ diff --git a/docs/contributing/issue_triage.md b/docs/contributing/issue_triage.md index 780b2b5e959..9b987fb2425 100644 --- a/docs/contributing/issue_triage.md +++ b/docs/contributing/issue_triage.md @@ -116,7 +116,9 @@ it's totally cool to go with it instead. For most issues, closing the issue manually or automatically after a resolving PR is ideal. For bug reports specifically, if the bug has already been fixed, try to check in -with the issue opener that their specific case has been resolved before closing. +with the issue opener that their specific case has been resolved before closing. Note +that we close issues as soon as they're fixed in the `main` branch. This doesn't +necessarily mean they've been released yet. Design and enhancement issues should be also closed when it's clear the proposed change won't be implemented, whether that has been determined after a lot of discussion or just @@ -146,7 +148,7 @@ Here's some of the most common issues and also pre-made responses you can use: ```text Black used to remove the trailing comma if the expression fits in a single line, but this was changed by #826 and #1288. Now a trailing comma tells Black to always explode the expression. This change was made mostly for the cases where you _know_ a collection or whatever will grow in the future. Having it always exploded as one element per line reduces diff noise when adding elements. Before the "magic trailing comma" feature, you couldn't anticipate a collection's growth reliably since collections that fitted in one line were ruthlessly collapsed regardless of your intentions. One of Black's goals is reducing diff noise, so this was a good pragmatic change. -So no, this is not a bug, but an intended feature. Anyway, [here's the documentation](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#the-magic-trailing-comma) on the "magic trailing comma". Hopefully that helps and sorry for the possible confusion. +So no, this is not a bug, but an intended feature. Anyway, [here's the documentation](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#the-magic-trailing-comma) on the "magic trailing comma", including the ability to skip this functionality with the `--skip-magic-trailing-comma` option. Hopefully that helps solve the possible confusion. ``` ### "Black formatted code is violating Flake8's E203!" From b8450b9faee10f3a0a63378b58adde60a27964e0 Mon Sep 17 00:00:00 2001 From: Hadi Alqattan Date: Sun, 16 May 2021 20:51:27 +0300 Subject: [PATCH 79/81] Fix: black only respects the root gitignore. (#2225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit history before merge: Black now respects .gitignore files in all levels, not only root/.gitignore file (apply .gitignore rules like git does). * Fix: typo * Fix: respect .gitignore files in all levels. * Add: CHANGELOG note. * Fix: TypeError: unsupported operand type(s) for +: 'NoneType' and 'PathSpec' * Update docs. * Fix: no parent .gitignore * Add a comment since the if expression is a bit hard to understand * Update tests - conver no parent .gitignore case. * Use main's Pipfile.lock instead The original changes in Pipfile.lock are whitespace only. The changes turned the JSON's file indentation from 4 to 2. Effectively this happened: `json.dumps(json.loads(old_pipfile_lock), indent=2) + "\n"`. Just using main's Pipfile.lock instead of undoing the changes because 1) I don't know how to do that easily and quickly, and 2) there's a merge conflict. Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> * Merge remote-tracking branch 'upstream/main' into i1730 … conflicts for days ay? --- CHANGES.md | 2 ++ .../file_collection_and_discovery.md | 3 +- src/black/files.py | 4 ++- .../nested_gitignore_tests/pyproject.toml | 3 ++ .../nested_gitignore_tests/root/.gitignore | 1 + tests/data/nested_gitignore_tests/root/a.py | 1 + tests/data/nested_gitignore_tests/root/b.py | 1 + tests/data/nested_gitignore_tests/root/c.py | 1 + .../root/child/.gitignore | 1 + .../nested_gitignore_tests/root/child/a.py | 1 + .../nested_gitignore_tests/root/child/b.py | 1 + .../nested_gitignore_tests/root/child/c.py | 1 + tests/data/nested_gitignore_tests/x.py | 0 tests/test_black.py | 29 ++++++++++++++++++- 14 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 tests/data/nested_gitignore_tests/pyproject.toml create mode 100644 tests/data/nested_gitignore_tests/root/.gitignore create mode 100644 tests/data/nested_gitignore_tests/root/a.py create mode 100644 tests/data/nested_gitignore_tests/root/b.py create mode 100644 tests/data/nested_gitignore_tests/root/c.py create mode 100644 tests/data/nested_gitignore_tests/root/child/.gitignore create mode 100644 tests/data/nested_gitignore_tests/root/child/a.py create mode 100644 tests/data/nested_gitignore_tests/root/child/b.py create mode 100644 tests/data/nested_gitignore_tests/root/child/c.py create mode 100644 tests/data/nested_gitignore_tests/x.py diff --git a/CHANGES.md b/CHANGES.md index f2c795440f8..603554cd8b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ ### _Black_ +- Respect `.gitignore` files in all levels, not only `root/.gitignore` file (apply + `.gitignore` rules like `git` does) (#2225) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) ### _Blackd_ diff --git a/docs/usage_and_configuration/file_collection_and_discovery.md b/docs/usage_and_configuration/file_collection_and_discovery.md index 54c76cd9a0f..1f436182dda 100644 --- a/docs/usage_and_configuration/file_collection_and_discovery.md +++ b/docs/usage_and_configuration/file_collection_and_discovery.md @@ -30,8 +30,7 @@ then write the above files to `.cache/black//`. ## .gitignore If `--exclude` is not set, _Black_ will automatically ignore files and directories in -`.gitignore` file, if present. The `.gitignore` file must be in the project root to be -used and nested `.gitignore` aren't supported. +`.gitignore` file(s), if present. If you want _Black_ to continue using `.gitignore` while also configuring the exclusion rules, please use `--extend-exclude`. diff --git a/src/black/files.py b/src/black/files.py index 1be560643a1..a0c92e8c415 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -204,6 +204,8 @@ def gen_python_files( continue if child.is_dir(): + # If gitignore is None, gitignore usage is disabled, while a Falsey + # gitignore is when the directory doesn't have a .gitignore file. yield from gen_python_files( child.iterdir(), root, @@ -212,7 +214,7 @@ def gen_python_files( extend_exclude, force_exclude, report, - gitignore, + gitignore + get_gitignore(child) if gitignore is not None else None, ) elif child.is_file(): diff --git a/tests/data/nested_gitignore_tests/pyproject.toml b/tests/data/nested_gitignore_tests/pyproject.toml new file mode 100644 index 00000000000..9ba7ec26980 --- /dev/null +++ b/tests/data/nested_gitignore_tests/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/tests/data/nested_gitignore_tests/root/.gitignore b/tests/data/nested_gitignore_tests/root/.gitignore new file mode 100644 index 00000000000..2987e7bb646 --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/.gitignore @@ -0,0 +1 @@ +a.py diff --git a/tests/data/nested_gitignore_tests/root/a.py b/tests/data/nested_gitignore_tests/root/a.py new file mode 100644 index 00000000000..7135cfd187c --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/a.py @@ -0,0 +1 @@ +# should be excluded (root/.gitignore) diff --git a/tests/data/nested_gitignore_tests/root/b.py b/tests/data/nested_gitignore_tests/root/b.py new file mode 100644 index 00000000000..bdeeca3c602 --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/b.py @@ -0,0 +1 @@ +# should be included diff --git a/tests/data/nested_gitignore_tests/root/c.py b/tests/data/nested_gitignore_tests/root/c.py new file mode 100644 index 00000000000..bdeeca3c602 --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/c.py @@ -0,0 +1 @@ +# should be included diff --git a/tests/data/nested_gitignore_tests/root/child/.gitignore b/tests/data/nested_gitignore_tests/root/child/.gitignore new file mode 100644 index 00000000000..6df81dd798e --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/child/.gitignore @@ -0,0 +1 @@ +b.py diff --git a/tests/data/nested_gitignore_tests/root/child/a.py b/tests/data/nested_gitignore_tests/root/child/a.py new file mode 100644 index 00000000000..7135cfd187c --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/child/a.py @@ -0,0 +1 @@ +# should be excluded (root/.gitignore) diff --git a/tests/data/nested_gitignore_tests/root/child/b.py b/tests/data/nested_gitignore_tests/root/child/b.py new file mode 100644 index 00000000000..c91d47946e6 --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/child/b.py @@ -0,0 +1 @@ +# should be excluded (child/.gitignore) diff --git a/tests/data/nested_gitignore_tests/root/child/c.py b/tests/data/nested_gitignore_tests/root/child/c.py new file mode 100644 index 00000000000..bdeeca3c602 --- /dev/null +++ b/tests/data/nested_gitignore_tests/root/child/c.py @@ -0,0 +1 @@ +# should be included diff --git a/tests/data/nested_gitignore_tests/x.py b/tests/data/nested_gitignore_tests/x.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_black.py b/tests/test_black.py index 5ab25cd1601..098a9ec9157 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1406,7 +1406,7 @@ def test_include_exclude(self) -> None: ) self.assertEqual(sorted(expected), sorted(sources)) - def test_gitingore_used_as_default(self) -> None: + def test_gitignore_used_as_default(self) -> None: path = Path(THIS_DIR / "data" / "include_exclude_tests") include = re.compile(r"\.pyi?$") extend_exclude = re.compile(r"/exclude/") @@ -1703,6 +1703,33 @@ def test_gitignore_exclude(self) -> None: ) self.assertEqual(sorted(expected), sorted(sources)) + def test_nested_gitignore(self) -> None: + path = Path(THIS_DIR / "data" / "nested_gitignore_tests") + include = re.compile(r"\.pyi?$") + exclude = re.compile(r"") + root_gitignore = black.files.get_gitignore(path) + report = black.Report() + expected: List[Path] = [ + Path(path / "x.py"), + Path(path / "root/b.py"), + Path(path / "root/c.py"), + Path(path / "root/child/c.py"), + ] + this_abs = THIS_DIR.resolve() + sources = list( + black.gen_python_files( + path.iterdir(), + this_abs, + include, + exclude, + None, + None, + report, + root_gitignore, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + def test_empty_include(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" report = black.Report() From 921c24af8032416101f532e148e6a382d28e28c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Hild=C3=A9n?= Date: Mon, 17 May 2021 21:38:43 +0300 Subject: [PATCH 80/81] Make Prettier preserve line ending type (#2244) Why? The default in Prettier 2.0 was [changed](https://prettier.io/docs/en/options.html#end-of-line) from `auto` to `LF`. This makes development on Windows awkward, because every file is marked with changes both by Prettier and then by Git regardless of repository line ending settings, making committing harder than it should be. --- Aside from that: I noticed that runnin pre-commit manually seems to add line endings to symlink files, but they disappear when actually committing. Don't know if that's a known.. quirk..(?) or not. --- Commit history before merge: * Make Prettier preserve line ending type * Move options to .prettierrc --- .pre-commit-config.yaml | 1 - .prettierrc.yaml | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .prettierrc.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e461aa39c9d..2527c4d42ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,3 @@ repos: rev: v2.2.1 hooks: - id: prettier - args: [--prose-wrap=always, --print-width=88] diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 00000000000..beda5ba4da8 --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,3 @@ +proseWrap: always +printWidth: 88 +endOfLine: auto From 7190d4f6c0dfcc7fce9adfffc1afc7254a01eae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Hild=C3=A9n?= Date: Mon, 17 May 2021 21:47:34 +0300 Subject: [PATCH 81/81] Fix test requirements file name (#2245) --- docs/contributing/the_basics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index d36b17eff31..d61f3ec45b6 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -22,7 +22,7 @@ from the cloned _Black_ repo. It will do the correct thing. Non pipenv install works too: ```console -$ pip install -r test_requirements +$ pip install -r test_requirements.txt $ pip install -e .[d] ```