Merge branch 'main' into no-changing-type-annotation-to-tuple

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,36 @@
+<!-- Hello! Thanks for submitting a PR. To help make things go a bit more
+     smoothly we would appreciate that you go through this template. -->
+
+### Description
+
+<!-- Good things to put here include: reasoning for the change (please link
+     any relevant issues!), any noteworthy (or hacky) choices to be aware of,
+     or what the problem resolved here looked like ... we won't mind a ranty
+     story :) -->
+
+### Checklist - did you ...
+
+<!-- If any of the following items aren't relevant for your contribution
+     please still tick them so we know you've gone through the checklist.
+
+    All user-facing changes should get an entry. Otherwise, signal to us
+    this should get the magical label to silence the CHANGELOG entry check.
+    Tests are required for bugfixes and new features. Documentation changes
+    are necessary for formatting and most enhancement changes. -->
+
+- [ ] Add a CHANGELOG entry if necessary?
+- [ ] Add / update tests if necessary?
+- [ ] Add new / update outdated documentation?
+
+<!-- Just as a reminder, everyone in all psf/black spaces including PRs
+     must follow the PSF Code of Conduct (link below).
+
+     Finally, once again thanks for your time and effort. If you have any
+     feedback in regards to your experience contributing here, please
+     let us know!
+
+     Helpful links:
+
+      PSF COC: https://www.python.org/psf/conduct/
+      Contributing docs: https://black.readthedocs.io/en/latest/contributing/index.html
+      Chat on Python Discord: https://discord.gg/RtVdv86PrH -->

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
         additional_dependencies: [flake8-bugbear]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.902
+    rev: v0.910
     hooks:
       - id: mypy
         exclude: ^docs/conf.py
@@ -31,7 +31,7 @@ repos:
           - platformdirs >= 2.1.0
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.3.1
+    rev: v2.3.2
     hooks:
       - id: prettier
         exclude: ^Pipfile\.lock

CHANGES.md

@@ -9,6 +9,12 @@
 - Present a more user-friendly error if .gitignore is invalid (#2414)
 - Avoid changing a function return type annotation's type to a tuple by adding a
   trailing comma (#2384)
+- The failsafe for accidentally added backslashes in f-string expressions has been
+  hardened to handle more edge cases during quote normalization (#2437)
+
+### _Blackd_
+
+- Replace sys.exit(-1) with raise ImportError (#2440)
 
 ### Integrations
 

Pipfile

@@ -4,23 +4,38 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 
 [dev-packages]
-Sphinx = ">=3.1.2"
-coverage = "*"
-docutils = "==0.15"  # not a direct dependency, see https://github.com/pypa/pipenv/issues/3865
-flake8 = "*"
+# Testing related requirements.
+coverage = ">= 5.3"
+pytest = " >= 6.1.1"
+pytest-mock = ">= 3.3.1"
+pytest-cases = ">= 2.3.0"
+pytest-xdist = ">= 2.2.1"
+pytest-cov = ">= 2.11.1"
+parameterized = ">= 0.7.4"
+tox = "*"
+
+# Linting related requirements.
+pre-commit = ">=2.9.2"
+flake8 = ">=3.9.2"
 flake8-bugbear = "*"
-mypy = ">=0.812"
+mypy = ">=0.910"
 types-dataclasses = ">=0.1.3"
 types-typed-ast = ">=1.4.1"
-pre-commit = "*"
-readme_renderer = "*"
-MyST-Parser = ">=0.13.7"
+
+# Documentation related requirements.
+Sphinx = ">=4.1.2"
+MyST-Parser = ">=0.15.1"
 sphinxcontrib-programoutput = ">=0.17"
-sphinx-copybutton = ">=0.3.0"
+sphinx-copybutton = ">=0.4.0"
+docutils = "==0.17.1"  # not a direct dependency, see https://github.com/pypa/pipenv/issues/3865
+
+# Packaging related requirements.
 setuptools = ">=39.2.0"
 setuptools-scm = "*"
 twine = ">=1.11.0"
 wheel = ">=0.31.1"
+readme_renderer = "*"
+
 black = {editable = true, extras = ["d", "jupyter"], path = "."}
 
 [packages]

docs/requirements.txt

@@ -1,6 +1,6 @@
 # Used by ReadTheDocs; pinned requirements for stability.
 
-MyST-Parser==0.14.0
-Sphinx==3.5.4
+myst-parser==0.15.1
+Sphinx==4.1.2
 sphinxcontrib-programoutput==0.17
-sphinx_copybutton==0.3.1
+sphinx_copybutton==0.4.0

src/black/strings.py

@@ -190,9 +190,9 @@ def normalize_string_quotes(s: str) -> str:
     if "f" in prefix.casefold():
         matches = re.findall(
             r"""
-            (?:[^{]|^)\{  # start of the string or a non-{ followed by a single {
+            (?:(?<!\{)|^)\{  # start of the string or a non-{ followed by a single {
                 ([^{].*?)  # contents of the brackets except if begins with {{
-            \}(?:[^}]|$)  # A } followed by end of the string or a non-}
+            \}(?:(?!\})|$)  # A } followed by end of the string or a non-}
             """,
             new_body,
             re.VERBOSE,

src/black_primer/lib.py

@@ -12,12 +12,23 @@
 from subprocess import CalledProcessError
 from sys import version_info
 from tempfile import TemporaryDirectory
-from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Tuple
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 from urllib.parse import urlparse
 
 import click
 
 
+TEN_MINUTES_SECONDS = 600
 WINDOWS = system() == "Windows"
 BLACK_BINARY = "black.exe" if WINDOWS else "black"
 GIT_BINARY = "git.exe" if WINDOWS else "git"
@@ -39,7 +50,7 @@ class Results(NamedTuple):
 
 async def _gen_check_output(
     cmd: Sequence[str],
-    timeout: float = 600,
+    timeout: float = TEN_MINUTES_SECONDS,
     env: Optional[Dict[str, str]] = None,
     cwd: Optional[Path] = None,
     stdin: Optional[bytes] = None,
@@ -113,6 +124,21 @@ def analyze_results(project_count: int, results: Results) -> int:
     return results.stats["failed"]
 
 
+def _flatten_cli_args(cli_args: List[Union[Sequence[str], str]]) -> List[str]:
+    """Allow a user to put long arguments into a list of strs
+    to make the JSON human readable"""
+    flat_args = []
+    for arg in cli_args:
+        if isinstance(arg, str):
+            flat_args.append(arg)
+            continue
+
+        args_as_str = "".join(arg)
+        flat_args.append(args_as_str)
+
+    return flat_args
+
+
 async def black_run(
     project_name: str,
     repo_path: Optional[Path],
@@ -131,7 +157,7 @@ async def black_run(
     stdin_test = project_name.upper() == "STDIN"
     cmd = [str(which(BLACK_BINARY))]
     if "cli_arguments" in project_config and project_config["cli_arguments"]:
-        cmd.extend(project_config["cli_arguments"])
+        cmd.extend(_flatten_cli_args(project_config["cli_arguments"]))
     cmd.append("--check")
     if not no_diff:
         cmd.append("--diff")
@@ -141,9 +167,16 @@ async def black_run(
     if stdin_test:
         cmd.append("-")
         stdin = repo_path.read_bytes()
+    elif "base_path" in project_config:
+        cmd.append(project_config["base_path"])
     else:
         cmd.append(".")
 
+    timeout = (
+        project_config["timeout_seconds"]
+        if "timeout_seconds" in project_config
+        else TEN_MINUTES_SECONDS
+    )
     with TemporaryDirectory() as tmp_path:
         # Prevent reading top-level user configs by manipulating environment variables
         env = {
@@ -154,8 +187,9 @@ async def black_run(
 
         cwd_path = repo_path.parent if stdin_test else repo_path
         try:
+            LOG.debug(f"Running black for {project_name}: {' '.join(cmd)}")
             _stdout, _stderr = await _gen_check_output(
-                cmd, cwd=cwd_path, env=env, stdin=stdin
+                cmd, cwd=cwd_path, env=env, stdin=stdin, timeout=timeout
             )
         except asyncio.TimeoutError:
             results.stats["failed"] += 1

src/black_primer/primer.json

@@ -1,5 +1,5 @@
 {
-  "configuration_format_version": 20200509,
+  "configuration_format_version": 20210815,
   "projects": {
     "STDIN": {
       "cli_arguments": ["--experimental-string-processing"],
@@ -36,6 +36,37 @@
       "long_checkout": false,
       "py_versions": ["all"]
     },
+    "cpython": {
+      "disabled": true,
+      "disabled_reason": "To big / slow for GitHub Actions but handy to keep config to use manually or in some other CI in the future",
+      "base_path": "Lib",
+      "cli_arguments": [
+        "--experimental-string-processing",
+        "--extend-exclude",
+        [
+          "Lib/lib2to3/tests/data/different_encoding.py",
+          "|Lib/lib2to3/tests/data/false_encoding.py",
+          "|Lib/lib2to3/tests/data/py2_test_grammar.py",
+          "|Lib/test/bad_coding.py",
+          "|Lib/test/bad_coding2.py",
+          "|Lib/test/badsyntax_3131.py",
+          "|Lib/test/badsyntax_pep3120.py",
+          "|Lib/test/test_base64.py",
+          "|Lib/test/test_exceptions.py",
+          "|Lib/test/test_grammar.py",
+          "|Lib/test/test_named_expressions.py",
+          "|Lib/test/test_patma.py",
+          "|Lib/test/test_tokenize.py",
+          "|Lib/test/test_xml_etree.py",
+          "|Lib/traceback.py"
+        ]
+      ],
+      "expect_formatting_changes": true,
+      "git_clone_url": "https://github.com/python/cpython.git",
+      "long_checkout": false,
+      "py_versions": ["3.9", "3.10"],
+      "timeout_seconds": 900
+    },
     "django": {
       "cli_arguments": [
         "--experimental-string-processing",

src/blackd/__init__.py

@@ -1,6 +1,5 @@
 import asyncio
 import logging
-import sys
 from concurrent.futures import Executor, ProcessPoolExecutor
 from datetime import datetime
 from functools import partial
@@ -11,13 +10,11 @@
     from aiohttp import web
     import aiohttp_cors
 except ImportError as ie:
-    print(
+    raise ImportError(
         f"aiohttp dependency is not installed: {ie}. "
         + "Please re-install black with the '[d]' extra install "
-        + "to obtain aiohttp_cors: `pip install black[d]`",
-        file=sys.stderr,
-    )
-    sys.exit(-1)
+        + "to obtain aiohttp_cors: `pip install black[d]`"
+    ) from None
 
 import black
 from black.concurrency import maybe_install_uvloop

tests/data/string_quotes.py

@@ -51,6 +51,11 @@
 '\'{z}\' {y * " "}'
 '{y * x} \'{z}\''
 
+# We must bail out if changing the quotes would introduce backslashes in f-string
+# expressions. xref: https://github.com/psf/black/issues/2348
+f"\"{b}\"{' ' * (long-len(b)+1)}: \"{sts}\",\n"
+f"\"{a}\"{'hello' * b}\"{c}\""
+
 # output
 
 """"""
@@ -100,3 +105,8 @@
 f"{y * x} '{z}'"
 "'{z}' {y * \" \"}"
 "{y * x} '{z}'"
+
+# We must bail out if changing the quotes would introduce backslashes in f-string
+# expressions. xref: https://github.com/psf/black/issues/2348
+f"\"{b}\"{' ' * (long-len(b)+1)}: \"{sts}\",\n"
+f"\"{a}\"{'hello' * b}\"{c}\""

tests/test_primer.py

@@ -146,6 +146,11 @@ def test_black_run(self) -> None:
             )
         self.assertEqual(2, results.stats["failed"])
 
+    def test_flatten_cli_args(self) -> None:
+        fake_long_args = ["--arg", ["really/", "|long", "|regex", "|splitup"], "--done"]
+        expected = ["--arg", "really/|long|regex|splitup", "--done"]
+        self.assertEqual(expected, lib._flatten_cli_args(fake_long_args))
+
     @event_loop()
     def test_gen_check_output(self) -> None:
         loop = asyncio.get_event_loop()
@@ -184,6 +189,8 @@ def test_git_checkout_or_rebase(self) -> None:
     @patch("sys.stdout", new_callable=StringIO)
     @event_loop()
     def test_process_queue(self, mock_stdout: Mock) -> None:
+        """Test the process queue on primer itself
+        - If you have non black conforming formatting in primer itself this can fail"""
         loop = asyncio.get_event_loop()
         config_path = Path(lib.__file__).parent / "primer.json"
         with patch("black_primer.lib.git_checkout_or_rebase", return_false):