diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e8d232c8b34..cb64cf9325d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,41 +6,58 @@ labels: "T: bug" assignees: "" --- + + **Describe the bug** **To Reproduce** - -For example: -1. Take this file '...' -1. Run _Black_ on it with these arguments '...' -1. See error --> +For example, take this code: -**Expected behavior** +```python +this = "code" +``` - +And run it with these arguments: -**Environment (please complete the following information):** +```sh +$ black file.py --target-version py39 +``` -- Version: -- OS and Python version: +The resulting error is: -**Does this bug also happen on main?** +> cannot format file.py: INTERNAL ERROR: ... - + + +**Environment** + + + +- Black's version: +- OS and Python version: **Additional context** diff --git a/CHANGES.md b/CHANGES.md index 06f1020e191..28f13e6dcba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,16 @@ # Change Log -## Unreleased +## 21.10b0 ### _Black_ +- Document stability policy, that will apply for non-beta releases (#2529) - Add new `--workers` parameter (#2514) +- Fixed feature detection for positional-only arguments in lambdas (#2532) - Bumped typed-ast version minimum to 1.4.3 for 3.10 compatiblity (#2519) +- Fixed a Python 3.10 compatibility issue where the loop argument was still being passed + even though it has been removed (#2580) +- Deprecate Python 2 formatting support (#2523) - `--verbose` enhancements (project root, how was config found, root relative paths) (#2526) @@ -14,9 +19,16 @@ - Remove dependency on aiohttp-cors (#2500) - Bump required aiohttp version to 3.7.4 (#2509) +### _Black-Primer_ + +- Add primer support for --projects (#2555) +- Print primer summary after individual failures (#2570) + ### Integrations - Allow to pass `target_version` in the vim plugin (#1319) +- Install build tools in docker file and use multi-stage build to keep the image size + down (#2582) ## 21.9b0 diff --git a/Dockerfile b/Dockerfile index 9542479eca5..c393e29f632 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,17 @@ -FROM python:3-slim +FROM python:3-slim AS builder RUN mkdir /src COPY . /src/ RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ - && apt update && apt install -y git \ + # Install build tools to compile dependencies that don't have prebuilt wheels + && apt update && apt install -y git build-essential \ && cd /src \ - && pip install --no-cache-dir .[colorama,d] \ - && rm -rf /src \ - && apt remove -y git \ - && apt autoremove -y \ - && rm -rf /var/lib/apt/lists/* + && pip install --user --no-cache-dir .[colorama,d] + +FROM python:3-slim + +# copy only Python packages to limit the image size +COPY --from=builder /root/.local /root/.local +ENV PATH=/root/.local/bin:$PATH CMD ["black"] diff --git a/README.md b/README.md index 7bf0ed8d16f..f9061c33863 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Ked many more. The following organizations use _Black_: Facebook, Dropbox, Mozilla, Quora, Duolingo, -QuantumBlack. +QuantumBlack, Tesla. Are we missing anyone? Let us know. diff --git a/docs/faq.md b/docs/faq.md index c361addf7ae..77f9df51fd4 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -31,6 +31,10 @@ pragmatism. However, _Black_ is still in beta so style changes are both planned still proposed on the issue tracker. See [The Black Code Style](the_black_code_style/index.rst) for more details. +Starting in 2022, the formatting output will be stable for the releases made in the same +year (other than unintentional bugs). It is possible to opt-in to the latest formatting +styles, using the `--future` flag. + ## Why is my file not formatted? Most likely because it is ignored in `.gitignore` or excluded with configuration. See @@ -70,10 +74,16 @@ disabled-by-default counterpart W504. E203 should be disabled while changes are ## Does Black support Python 2? +```{warning} +Python 2 support has been deprecated since 21.10b0. + +This support will be dropped in the first stable release, expected for January 2022. +See [The Black Code Style](the_black_code_style/index.rst) for details. +``` + For formatting, yes! [Install](getting_started.md#installation) with the `python2` extra -to format Python 2 files too! There are no current plans to drop support, but most -likely it is bound to happen. Sometime. Eventually. In terms of running _Black_ though, -Python 3.6 or newer is required. +to format Python 2 files too! In terms of running _Black_ though, Python 3.6 or newer is +required. ## Why does my linter or typechecker complain after I format my code? diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 6a1aa363d2b..cf0ef1dfed9 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -7,7 +7,7 @@ Use [pre-commit](https://pre-commit.com/). Once you ```yaml repos: - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 21.10b0 hooks: - id: black # It is recommended to specify the latest version of Python diff --git a/docs/requirements.txt b/docs/requirements.txt index 4c5b700412a..296efc5cc84 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==0.15.1 -Sphinx==4.1.2 +Sphinx==4.2.0 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.4.0 diff --git a/docs/the_black_code_style/index.rst b/docs/the_black_code_style/index.rst index 4693437be8b..7c2e1753937 100644 --- a/docs/the_black_code_style/index.rst +++ b/docs/the_black_code_style/index.rst @@ -9,9 +9,33 @@ The Black Code Style *Black* is a PEP 8 compliant opinionated formatter with its own style. -It should be noted that while keeping the style unchanged throughout releases is a -goal, the *Black* code style isn't set in stone. Sometimes it's modified in response to -user feedback or even changes to the Python language! +While keeping the style unchanged throughout releases has always been a goal, +the *Black* code style isn't set in stone. It evolves to accomodate for new features +in the Python language and, ocassionally, in response to user feedback. + +Stability Policy +---------------- + +The following policy applies for the *Black* code style, in non pre-release +versions of *Black*: + +- The same code, formatted with the same options, will produce the same + output for all releases in a given calendar year. + + This means projects can safely use `black ~= 22.0` without worrying about + major formatting changes disrupting their project in 2022. We may still + fix bugs where *Black* crashes on some code, and make other improvements + that do not affect formatting. + +- The first release in a new calendar year *may* contain formatting changes, + although these will be minimised as much as possible. This is to allow for + improved formatting enabled by newer Python language syntax as well as due + to improvements in the formatting logic. + +- The ``--future`` flag is exempt from this policy. There are no guarentees + around the stability of the output with that flag passed into *Black*. This + flag is intended for allowing experimentation with the proposed changes to + the *Black* code style. Documentation for both the current and future styles can be found: diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 49268b44f7c..533c213a4da 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -173,7 +173,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, version 21.9b0 +black, version 21.10b0 ``` An option to require a specific version to be running is also provided. diff --git a/mypy.ini b/mypy.ini index 7e563e6f696..62c1c7fefaa 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,8 +22,7 @@ strict_optional=True warn_no_return=True warn_redundant_casts=True warn_unused_ignores=True -# Until we're not supporting 3.6 primer needs this -disallow_any_generics=False +disallow_any_generics=True # The following are off by default. Flip them on if you feel # adventurous. @@ -33,9 +32,10 @@ check_untyped_defs=True # No incremental mode cache_dir=/dev/null -[mypy-aiohttp.*] -follow_imports=skip -[mypy-black] -# The following is because of `patch_click()`. Remove when -# we drop Python 3.6 support. -warn_unused_ignores=False +[mypy-black_primer.*] +# Until we're not supporting 3.6 primer needs this +disallow_any_generics=False + +[mypy-tests.test_primer] +# Until we're not supporting 3.6 primer needs this +disallow_any_generics=False diff --git a/src/black/__init__.py b/src/black/__init__.py index a3d1183eac9..9b251200a91 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -171,7 +171,7 @@ def validate_regex( ctx: click.Context, param: click.Parameter, value: Optional[str], -) -> Optional[Pattern]: +) -> Optional[Pattern[str]]: try: return re_compile_maybe_verbose(value) if value is not None else None except re.error: @@ -389,10 +389,10 @@ def main( quiet: bool, verbose: bool, required_version: str, - include: Pattern, - exclude: Optional[Pattern], - extend_exclude: Optional[Pattern], - force_exclude: Optional[Pattern], + include: Pattern[str], + exclude: Optional[Pattern[str]], + extend_exclude: Optional[Pattern[str]], + force_exclude: Optional[Pattern[str]], stdin_filename: Optional[str], workers: int, src: Tuple[str, ...], @@ -789,7 +789,10 @@ async def schedule_formatting( sources_to_cache.append(src) report.done(src, changed, root) if cancelled: - await asyncio.gather(*cancelled, loop=loop, return_exceptions=True) + if sys.version_info >= (3, 7): + await asyncio.gather(*cancelled, return_exceptions=True) + else: + await asyncio.gather(*cancelled, loop=loop, return_exceptions=True) if sources_to_cache: write_cache(cache, sources_to_cache, mode) @@ -1084,6 +1087,15 @@ def f( versions = mode.target_versions else: versions = detect_target_versions(src_node) + + # TODO: fully drop support and this code hopefully in January 2022 :D + if TargetVersion.PY27 in mode.target_versions or versions == {TargetVersion.PY27}: + msg = ( + "DEPRECATION: Python 2 support will be removed in the first stable release" + "expected in January 2022." + ) + err(msg, fg="yellow", bold=True) + normalize_fmt_off(src_node) lines = LineGenerator( mode=mode, @@ -1126,7 +1138,7 @@ def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]: return tiow.read(), encoding, newline -def get_features_used(node: Node) -> Set[Feature]: +def get_features_used(node: Node) -> Set[Feature]: # noqa: C901 """Return a set of (relatively) new Python features used in this file. Currently looking for: @@ -1136,6 +1148,7 @@ def get_features_used(node: Node) -> Set[Feature]: - positional only arguments in function signatures and lambdas; - assignment expression; - relaxed decorator syntax; + - print / exec statements; """ features: Set[Feature] = set() for n in node.pre_order(): @@ -1149,7 +1162,11 @@ def get_features_used(node: Node) -> Set[Feature]: features.add(Feature.NUMERIC_UNDERSCORES) elif n.type == token.SLASH: - if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}: + if n.parent and n.parent.type in { + syms.typedargslist, + syms.arglist, + syms.varargslist, + }: features.add(Feature.POS_ONLY_ARGUMENTS) elif n.type == token.COLONEQUAL: @@ -1180,6 +1197,11 @@ def get_features_used(node: Node) -> Set[Feature]: if argch.type in STARS: features.add(feature) + elif n.type == token.PRINT_STMT: + features.add(Feature.PRINT_STMT) + elif n.type == token.EXEC_STMT: + features.add(Feature.EXEC_STMT) + return features @@ -1309,7 +1331,7 @@ def patch_click() -> None: """ try: from click import core - from click import _unicodefun # type: ignore + from click import _unicodefun except ModuleNotFoundError: return diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 69d79f534e8..24f67b62f06 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -42,9 +42,12 @@ def shutdown(loop: asyncio.AbstractEventLoop) -> None: for task in to_cancel: task.cancel() - loop.run_until_complete( - asyncio.gather(*to_cancel, loop=loop, return_exceptions=True) - ) + if sys.version_info >= (3, 7): + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + else: + loop.run_until_complete( + asyncio.gather(*to_cancel, loop=loop, return_exceptions=True) + ) finally: # `concurrent.futures.Future` objects cannot be cancelled once they # are already running. There might be some when the `shutdown()` happened. diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 63c8aafe35b..f10eaed4f3e 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -39,18 +39,18 @@ ) NON_PYTHON_CELL_MAGICS = frozenset( ( - "%%bash", - "%%html", - "%%javascript", - "%%js", - "%%latex", - "%%markdown", - "%%perl", - "%%ruby", - "%%script", - "%%sh", - "%%svg", - "%%writefile", + "bash", + "html", + "javascript", + "js", + "latex", + "markdown", + "perl", + "ruby", + "script", + "sh", + "svg", + "writefile", ) ) TOKEN_HEX = secrets.token_hex @@ -230,10 +230,11 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: cell_magic_finder.visit(tree) if cell_magic_finder.cell_magic is None: return src, replacements - if cell_magic_finder.cell_magic.header.split()[0] in NON_PYTHON_CELL_MAGICS: + if cell_magic_finder.cell_magic.name in NON_PYTHON_CELL_MAGICS: raise NothingChanged - mask = get_token(src, cell_magic_finder.cell_magic.header) - replacements.append(Replacement(mask=mask, src=cell_magic_finder.cell_magic.header)) + header = cell_magic_finder.cell_magic.header + mask = get_token(src, header) + replacements.append(Replacement(mask=mask, src=header)) return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements @@ -311,11 +312,26 @@ def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: ) +def _get_str_args(args: List[ast.expr]) -> List[str]: + str_args = [] + for arg in args: + assert isinstance(arg, ast.Str) + str_args.append(arg.s) + return str_args + + @dataclasses.dataclass(frozen=True) class CellMagic: - header: str + name: str + params: Optional[str] body: str + @property + def header(self) -> str: + if self.params: + return f"%%{self.name} {self.params}" + return f"%%{self.name}" + @dataclasses.dataclass class CellMagicFinder(ast.NodeVisitor): @@ -345,14 +361,8 @@ def visit_Expr(self, node: ast.Expr) -> None: and _is_ipython_magic(node.value.func) and node.value.func.attr == "run_cell_magic" ): - args = [] - for arg in node.value.args: - assert isinstance(arg, ast.Str) - args.append(arg.s) - header = f"%%{args[0]}" - if args[1]: - header += f" {args[1]}" - self.cell_magic = CellMagic(header=header, body=args[2]) + args = _get_str_args(node.value.args) + self.cell_magic = CellMagic(name=args[0], params=args[1], body=args[2]) self.generic_visit(node) @@ -404,12 +414,8 @@ def visit_Assign(self, node: ast.Assign) -> None: and _is_ipython_magic(node.value.func) and node.value.func.attr == "getoutput" ): - args = [] - for arg in node.value.args: - assert isinstance(arg, ast.Str) - args.append(arg.s) - assert args - src = f"!{args[0]}" + (arg,) = _get_str_args(node.value.args) + src = f"!{arg}" self.magics[node.value.lineno].append( OffsetAndMagic(node.value.col_offset, src) ) @@ -435,11 +441,7 @@ def visit_Expr(self, node: ast.Expr) -> None: and we look for instances of any of the latter. """ if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): - args = [] - for arg in node.value.args: - assert isinstance(arg, ast.Str) - args.append(arg.s) - assert args + args = _get_str_args(node.value.args) if node.value.func.attr == "run_line_magic": if args[0] == "pinfo": src = f"?{args[1]}" diff --git a/src/black/mode.py b/src/black/mode.py index 0b7624eaf8a..374c47a42eb 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -41,9 +41,17 @@ class Feature(Enum): RELAXED_DECORATORS = 10 FORCE_OPTIONAL_PARENTHESES = 50 + # temporary for Python 2 deprecation + PRINT_STMT = 200 + EXEC_STMT = 201 + VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = { - TargetVersion.PY27: {Feature.ASYNC_IDENTIFIERS}, + TargetVersion.PY27: { + Feature.ASYNC_IDENTIFIERS, + Feature.PRINT_STMT, + Feature.EXEC_STMT, + }, TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS}, TargetVersion.PY35: { diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 8360fc3c703..2395d35886a 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -1,13 +1,14 @@ # coding=utf8 import asyncio +import json import logging import sys from datetime import datetime from pathlib import Path from shutil import rmtree, which from tempfile import gettempdir -from typing import Any, Union, Optional +from typing import Any, List, Optional, Union import click @@ -42,12 +43,42 @@ def _handle_debug( return debug +def load_projects(config_path: Path) -> List[str]: + with open(config_path) as config: + return sorted(json.load(config)["projects"].keys()) + + +# Unfortunately does import time file IO - but appears to be the only +# way to get `black-primer --help` to show projects list +DEFAULT_PROJECTS = load_projects(DEFAULT_CONFIG) + + +def _projects_callback( + ctx: click.core.Context, + param: Optional[Union[click.core.Option, click.core.Parameter]], + projects: str, +) -> List[str]: + requested_projects = set(projects.split(",")) + available_projects = set( + DEFAULT_PROJECTS + if str(DEFAULT_CONFIG) == ctx.params["config"] + else load_projects(ctx.params["config"]) + ) + + unavailable = requested_projects - available_projects + if unavailable: + LOG.error(f"Projects not found: {unavailable}. Available: {available_projects}") + + return sorted(requested_projects & available_projects) + + async def async_main( config: str, debug: bool, keep: bool, long_checkouts: bool, no_diff: bool, + projects: List[str], rebase: bool, workdir: str, workers: int, @@ -66,6 +97,7 @@ async def async_main( config, work_path, workers, + projects, keep, long_checkouts, rebase, @@ -88,6 +120,8 @@ async def async_main( type=click.Path(exists=True), show_default=True, help="JSON config file path", + # Eager - because config path is used by other callback options + is_eager=True, ) @click.option( "--debug", @@ -116,6 +150,13 @@ async def async_main( show_default=True, help="Disable showing source file changes in black output", ) +@click.option( + "--projects", + default=",".join(DEFAULT_PROJECTS), + callback=_projects_callback, + show_default=True, + help="Comma separated list of projects to run", +) @click.option( "-R", "--rebase", diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index 7494ae6dc7d..13724f431ce 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -88,6 +88,18 @@ def analyze_results(project_count: int, results: Results) -> int: failed_pct = round(((results.stats["failed"] / project_count) * 100), 2) success_pct = round(((results.stats["success"] / project_count) * 100), 2) + if results.failed_projects: + click.secho("\nFailed projects:\n", bold=True) + + for project_name, project_cpe in results.failed_projects.items(): + print(f"## {project_name}:") + print(f" - Returned {project_cpe.returncode}") + if project_cpe.stderr: + print(f" - stderr:\n{project_cpe.stderr.decode('utf8')}") + if project_cpe.stdout: + print(f" - stdout:\n{project_cpe.stdout.decode('utf8')}") + print("") + click.secho("-- primer results 📊 --\n", bold=True) click.secho( f"{results.stats['success']} / {project_count} succeeded ({success_pct}%) ✅", @@ -110,16 +122,8 @@ def analyze_results(project_count: int, results: Results) -> int: ) if results.failed_projects: - click.secho("\nFailed projects:\n", bold=True) - - for project_name, project_cpe in results.failed_projects.items(): - print(f"## {project_name}:") - print(f" - Returned {project_cpe.returncode}") - if project_cpe.stderr: - print(f" - stderr:\n{project_cpe.stderr.decode('utf8')}") - if project_cpe.stdout: - print(f" - stdout:\n{project_cpe.stdout.decode('utf8')}") - print("") + failed = ", ".join(results.failed_projects.keys()) + click.secho(f"\nFailed projects: {failed}\n", bold=True) return results.stats["failed"] @@ -258,7 +262,7 @@ async def git_checkout_or_rebase( def handle_PermissionError( - func: Callable, path: Path, exc: Tuple[Any, Any, Any] + func: Callable[..., None], path: Path, exc: Tuple[Any, Any, Any] ) -> None: """ Handle PermissionError during shutil.rmtree. @@ -283,16 +287,16 @@ def handle_PermissionError( async def load_projects_queue( config_path: Path, + projects_to_run: List[str], ) -> Tuple[Dict[str, Any], asyncio.Queue]: """Load project config and fill queue with all the project names""" with config_path.open("r") as cfp: config = json.load(cfp) # TODO: Offer more options here - # e.g. Run on X random packages or specific sub list etc. - project_names = sorted(config["projects"].keys()) - queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names)) - for project in project_names: + # e.g. Run on X random packages etc. + queue: asyncio.Queue = asyncio.Queue(maxsize=len(projects_to_run)) + for project in projects_to_run: await queue.put(project) return config, queue @@ -365,6 +369,7 @@ async def process_queue( config_file: str, work_path: Path, workers: int, + projects_to_run: List[str], keep: bool = False, long_checkouts: bool = False, rebase: bool = False, @@ -383,7 +388,7 @@ async def process_queue( results.stats["success"] = 0 results.stats["wrong_py_ver"] = 0 - config, queue = await load_projects_queue(Path(config_file)) + config, queue = await load_projects_queue(Path(config_file), projects_to_run) project_count = queue.qsize() s = "" if project_count == 1 else "s" LOG.info(f"{project_count} project{s} to run Black over") diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 1e0dec9c714..349ba8023a2 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -74,6 +74,9 @@ COLONEQUAL: Final = 59 N_TOKENS: Final = 60 NT_OFFSET: Final = 256 +# temporary for Python 2 deprecation +PRINT_STMT: Final = 316 +EXEC_STMT: Final = 288 # --end constants-- tok_name: Final[Dict[int, str]] = {} diff --git a/tests/test_black.py b/tests/test_black.py index 213f58c5dea..aa5f423ed0e 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -50,6 +50,7 @@ DATA_DIR, DEFAULT_MODE, DETERMINISTIC_HEADER, + PROJECT_ROOT, PY36_VERSIONS, THIS_DIR, BlackBaseTestCase, @@ -805,6 +806,10 @@ def test_get_features_used(self) -> None: self.assertEqual(black.get_features_used(node), set()) node = black.lib2to3_parse(expected) self.assertEqual(black.get_features_used(node), set()) + node = black.lib2to3_parse("lambda a, /, b: ...") + self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) + node = black.lib2to3_parse("def fn(a, /, b): ...") + self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) def test_get_future_imports(self) -> None: node = black.lib2to3_parse("\n") @@ -1408,14 +1413,14 @@ def test_docstring_reformat_for_py27(self) -> None: ) expected = 'def foo():\n """Testing\n Testing"""\n print "Foo"\n' - result = CliRunner().invoke( + result = BlackRunner().invoke( black.main, ["-", "-q", "--target-version=py27"], input=BytesIO(source), ) self.assertEqual(result.exit_code, 0) - actual = result.output + actual = result.stdout self.assertFormatEqual(actual, expected) @staticmethod @@ -1520,9 +1525,11 @@ def test_code_option_config(self) -> None: """ with patch.object(black, "parse_pyproject_toml", return_value={}) as parse: args = ["--code", "print"] - CliRunner().invoke(black.main, args) + # This is the only directory known to contain a pyproject.toml + with change_directory(PROJECT_ROOT): + CliRunner().invoke(black.main, args) + pyproject_path = Path(Path.cwd(), "pyproject.toml").resolve() - pyproject_path = Path(Path().cwd(), "pyproject.toml").resolve() assert ( len(parse.mock_calls) >= 1 ), "Expected config parse to be called with the current directory." @@ -1537,7 +1544,7 @@ def test_code_option_parent_config(self) -> None: Test that the code option finds the pyproject.toml in the parent directory. """ with patch.object(black, "parse_pyproject_toml", return_value={}) as parse: - with change_directory(Path("tests")): + with change_directory(THIS_DIR): args = ["--code", "print"] CliRunner().invoke(black.main, args) @@ -2022,11 +2029,28 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: ) +@pytest.mark.parametrize("explicit", [True, False], ids=["explicit", "autodetection"]) +def test_python_2_deprecation_with_target_version(explicit: bool) -> None: + args = [ + "--config", + str(THIS_DIR / "empty.toml"), + str(DATA_DIR / "python2.py"), + "--check", + ] + if explicit: + args.append("--target-version=py27") + with cache_dir(): + result = BlackRunner().invoke(black.main, args) + assert "DEPRECATION: Python 2 support will be removed" in result.stderr + + with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() -def tracefunc(frame: types.FrameType, event: str, arg: Any) -> Callable: +def tracefunc( + frame: types.FrameType, event: str, arg: Any +) -> Callable[[types.FrameType, str, Any], Any]: """Show function calls `from black/__init__.py` as they happen. Register this with `sys.settrace()` in a test you're debugging. diff --git a/tests/test_format.py b/tests/test_format.py index a659382092a..649c1572bee 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -93,6 +93,8 @@ "src/black/strings.py", "src/black/trans.py", "src/blackd/__init__.py", + "src/black_primer/cli.py", + "src/black_primer/lib.py", "src/blib2to3/pygram.py", "src/blib2to3/pytree.py", "src/blib2to3/pgen2/conv.py", diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 12f176c9341..ba460074e9a 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,4 +1,5 @@ -import pathlib +import re + from click.testing import CliRunner from black.handle_ipynb_magics import jupyter_dependencies_are_installed from black import ( @@ -8,11 +9,11 @@ format_file_contents, format_file_in_place, ) -import os import pytest from black import Mode from _pytest.monkeypatch import MonkeyPatch from py.path import local +from tests.util import DATA_DIR pytestmark = pytest.mark.jupyter pytest.importorskip("IPython", reason="IPython is an optional dependency") @@ -178,9 +179,7 @@ def test_empty_cell() -> None: def test_entire_notebook_empty_metadata() -> None: - with open( - os.path.join("tests", "data", "notebook_empty_metadata.ipynb"), "rb" - ) as fd: + with open(DATA_DIR / "notebook_empty_metadata.ipynb", "rb") as fd: content_bytes = fd.read() content = content_bytes.decode() result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) @@ -217,9 +216,7 @@ def test_entire_notebook_empty_metadata() -> None: def test_entire_notebook_trailing_newline() -> None: - with open( - os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), "rb" - ) as fd: + with open(DATA_DIR / "notebook_trailing_newline.ipynb", "rb") as fd: content_bytes = fd.read() content = content_bytes.decode() result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) @@ -268,9 +265,7 @@ def test_entire_notebook_trailing_newline() -> None: def test_entire_notebook_no_trailing_newline() -> None: - with open( - os.path.join("tests", "data", "notebook_no_trailing_newline.ipynb"), "rb" - ) as fd: + with open(DATA_DIR / "notebook_no_trailing_newline.ipynb", "rb") as fd: content_bytes = fd.read() content = content_bytes.decode() result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) @@ -319,9 +314,7 @@ def test_entire_notebook_no_trailing_newline() -> None: def test_entire_notebook_without_changes() -> None: - with open( - os.path.join("tests", "data", "notebook_without_changes.ipynb"), "rb" - ) as fd: + with open(DATA_DIR / "notebook_without_changes.ipynb", "rb") as fd: content_bytes = fd.read() content = content_bytes.decode() with pytest.raises(NothingChanged): @@ -329,7 +322,7 @@ def test_entire_notebook_without_changes() -> None: def test_non_python_notebook() -> None: - with open(os.path.join("tests", "data", "non_python_notebook.ipynb"), "rb") as fd: + with open(DATA_DIR / "non_python_notebook.ipynb", "rb") as fd: content_bytes = fd.read() content = content_bytes.decode() with pytest.raises(NothingChanged): @@ -342,23 +335,17 @@ def test_empty_string() -> None: def test_unparseable_notebook() -> None: - msg = ( - r"File 'tests[/\\]data[/\\]notebook_which_cant_be_parsed\.ipynb' " - r"cannot be parsed as valid Jupyter notebook\." - ) + path = DATA_DIR / "notebook_which_cant_be_parsed.ipynb" + msg = rf"File '{re.escape(str(path))}' cannot be parsed as valid Jupyter notebook\." with pytest.raises(ValueError, match=msg): - format_file_in_place( - pathlib.Path("tests") / "data/notebook_which_cant_be_parsed.ipynb", - fast=True, - mode=JUPYTER_MODE, - ) + format_file_in_place(path, fast=True, mode=JUPYTER_MODE) def test_ipynb_diff_with_change() -> None: result = runner.invoke( main, [ - os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), + str(DATA_DIR / "notebook_trailing_newline.ipynb"), "--diff", ], ) @@ -370,7 +357,7 @@ def test_ipynb_diff_with_no_change() -> None: result = runner.invoke( main, [ - os.path.join("tests", "data", "notebook_without_changes.ipynb"), + str(DATA_DIR / "notebook_without_changes.ipynb"), "--diff", ], ) @@ -383,7 +370,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. jupyter_dependencies_are_installed.cache_clear() - nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + nb = DATA_DIR / "notebook_trailing_newline.ipynb" tmp_nb = tmpdir / "notebook.ipynb" with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) @@ -405,7 +392,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. jupyter_dependencies_are_installed.cache_clear() - nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + nb = DATA_DIR / "notebook_trailing_newline.ipynb" tmp_nb = tmpdir / "notebook.ipynb" with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) @@ -423,7 +410,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( def test_ipynb_flag(tmpdir: local) -> None: - nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + nb = DATA_DIR / "notebook_trailing_newline.ipynb" tmp_nb = tmpdir / "notebook.a_file_extension_which_is_definitely_not_ipynb" with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) @@ -440,11 +427,11 @@ def test_ipynb_flag(tmpdir: local) -> None: def test_ipynb_and_pyi_flags() -> None: - nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + nb = DATA_DIR / "notebook_trailing_newline.ipynb" result = runner.invoke( main, [ - nb, + str(nb), "--pyi", "--ipynb", "--diff", diff --git a/tests/test_primer.py b/tests/test_primer.py index e7f99fdb0c2..9bb401574ca 100644 --- a/tests/test_primer.py +++ b/tests/test_primer.py @@ -9,9 +9,10 @@ from os import getpid from pathlib import Path from platform import system +from pytest import LogCaptureFixture from subprocess import CalledProcessError from tempfile import TemporaryDirectory, gettempdir -from typing import Any, Callable, Generator, Iterator, Tuple +from typing import Any, Callable, Iterator, List, Tuple, TypeVar from unittest.mock import Mock, patch from click.testing import CliRunner @@ -20,6 +21,14 @@ EXPECTED_ANALYSIS_OUTPUT = """\ + +Failed projects: + +## black: + - Returned 69 + - stdout: +Black didn't work + -- primer results 📊 -- 68 / 69 succeeded (98.55%) ✅ @@ -28,12 +37,7 @@ - 0 projects skipped due to Python version - 0 skipped due to long checkout -Failed projects: - -## black: - - Returned 69 - - stdout: -Black didn't work +Failed projects: black """ FAKE_PROJECT_CONFIG = { @@ -44,7 +48,9 @@ @contextmanager -def capture_stdout(command: Callable, *args: Any, **kwargs: Any) -> Generator: +def capture_stdout( + command: Callable[..., Any], *args: Any, **kwargs: Any +) -> Iterator[str]: old_stdout, sys.stdout = sys.stdout, StringIO() try: command(*args, **kwargs) @@ -87,6 +93,24 @@ async def return_zero(*args: Any, **kwargs: Any) -> int: return 0 +if sys.version_info >= (3, 9): + T = TypeVar("T") + Q = asyncio.Queue[T] +else: + T = Any + Q = asyncio.Queue + + +def collect(queue: Q) -> List[T]: + ret = [] + while True: + try: + item = queue.get_nowait() + ret.append(item) + except asyncio.QueueEmpty: + return ret + + class PrimerLibTests(unittest.TestCase): def test_analyze_results(self) -> None: fake_results = lib.Results( @@ -196,10 +220,25 @@ def test_process_queue(self, mock_stdout: Mock) -> None: with patch("black_primer.lib.git_checkout_or_rebase", return_false): with TemporaryDirectory() as td: return_val = loop.run_until_complete( - lib.process_queue(str(config_path), Path(td), 2) + lib.process_queue( + str(config_path), Path(td), 2, ["django", "pyramid"] + ) ) self.assertEqual(0, return_val) + @event_loop() + def test_load_projects_queue(self) -> None: + """Test the process queue on primer itself + - If you have non black conforming formatting in primer itself this can fail""" + loop = asyncio.get_event_loop() + config_path = Path(lib.__file__).parent / "primer.json" + + config, projects_queue = loop.run_until_complete( + lib.load_projects_queue(config_path, ["django", "pyramid"]) + ) + projects = collect(projects_queue) + self.assertEqual(projects, ["django", "pyramid"]) + class PrimerCLITests(unittest.TestCase): @event_loop() @@ -215,6 +254,7 @@ def test_async_main(self) -> None: "workdir": str(work_dir), "workers": 69, "no_diff": False, + "projects": "", } with patch("black_primer.cli.lib.process_queue", return_zero): return_val = loop.run_until_complete(cli.async_main(**args)) # type: ignore @@ -229,5 +269,23 @@ def test_help_output(self) -> None: self.assertEqual(result.exit_code, 0) +def test_projects(caplog: LogCaptureFixture) -> None: + with event_loop(): + runner = CliRunner() + result = runner.invoke(cli.main, ["--projects=STDIN,asdf"]) + assert result.exit_code == 0 + assert "1 / 1 succeeded" in result.output + assert "Projects not found: {'asdf'}" in caplog.text + + caplog.clear() + + with event_loop(): + runner = CliRunner() + result = runner.invoke(cli.main, ["--projects=fdsa,STDIN"]) + assert result.exit_code == 0 + assert "1 / 1 succeeded" in result.output + assert "Projects not found: {'fdsa'}" in caplog.text + + if __name__ == "__main__": unittest.main() diff --git a/tests/util.py b/tests/util.py index 84e98bb0fbd..8755111f7c5 100644 --- a/tests/util.py +++ b/tests/util.py @@ -9,7 +9,7 @@ import black from black.debug import DebugVisitor from black.mode import TargetVersion -from black.output import err, out +from black.output import diff, err, out THIS_DIR = Path(__file__).parent DATA_DIR = THIS_DIR / "data" @@ -47,6 +47,9 @@ def _assert_format_equal(expected: str, actual: str) -> None: except Exception as ve: err(str(ve)) + if actual != expected: + out(diff(expected, actual, "expected", "actual")) + assert actual == expected