diff --git a/.github/workflows/primer.yml b/.github/workflows/primer.yml index 5f41c301737..cd0b4bde6af 100644 --- a/.github/workflows/primer.yml +++ b/.github/workflows/primer.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[d]" + python -m pip install -e ".[d,jupyter]" - name: Primer run env: diff --git a/.gitignore b/.gitignore index ab796ce4cd0..f81bce8fd4e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ src/_black_version.py *.swp .hypothesis/ venv/ +.ipynb_checkpoints/ diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index de2eb674e0d..81848d7dcf7 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -7,3 +7,14 @@ minimum_pre_commit_version: 2.9.2 require_serial: true types_or: [python, pyi] +- id: black-jupyter + name: black-jupyter + description: + "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" + entry: black + language: python + language_version: python3 + minimum_pre_commit_version: 2.9.2 + require_serial: true + types_or: [python, pyi, jupyter] + additional_dependencies: [".[jupyter]"] diff --git a/CHANGES.md b/CHANGES.md index 6714a9d9eb2..a678aaefc2a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,10 @@ ## Unreleased -- Moved from `appdirs` dependency to `platformdirs` (#2375) +### _Black_ + +- Add support for formatting Jupyter Notebook files (#2357) +- Move from `appdirs` dependency to `platformdirs` (#2375) ## 21.7b0 diff --git a/Pipfile b/Pipfile index 8d75f30885d..2a836a28d69 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,6 @@ pathspec = ">=0.8.1" regex = ">=2020.1.8" tomli = ">=0.2.6, <2.0.0" typed-ast = "==1.4.2" -typing_extensions = {"python_version <" = "3.8","version >=" = "3.7.4"} +typing_extensions = {"python_version <" = "3.10","version >=" = "3.10.0.0"} black = {editable = true,extras = ["d"],path = "."} dataclasses = {"python_version <" = "3.7","version >" = "0.1.3"} diff --git a/README.md b/README.md index 7b87a03fc38..709478e1d58 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,8 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run. If you want to format Python 2 code as well, install with -`pip install black[python2]`. +`pip install black[python2]`. If you want to format Jupyter Notebooks, install with +`pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/docs/faq.md b/docs/faq.md index ac5ba937c1c..d7e6a16351f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -37,6 +37,29 @@ Most likely because it is ignored in `.gitignore` or excluded with configuration [file collection and discovery](usage_and_configuration/file_collection_and_discovery.md) for details. +## Why is my Jupyter Notebook cell not formatted? + +_Black_ is timid about formatting Jupyter Notebooks. Cells containing any of the +following will not be formatted: + +- automagics (e.g. `pip install black`) +- multiline magics, e.g.: + + ```python + %timeit f(1, \ + 2, \ + 3) + ``` + +- code which `IPython`'s `TransformerManager` would transform magics into, e.g.: + + ```python + get_ipython().system('ls') + ``` + +- invalid syntax, as it can't be safely distinguished from automagics in the absense of + a running `IPython` kernel. + ## Why are Flake8's E203 and W503 violated? Because they go against PEP 8. E203 falsely triggers on list diff --git a/docs/getting_started.md b/docs/getting_started.md index a509d34e903..c79dc607c4a 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -18,7 +18,8 @@ Also, you can try out _Black_ online for minimal fuss on the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run, but can format Python 2 code too. Python 2 support needs the `typed_ast` -dependency, which be installed with `pip install black[python2]`. +dependency, which be installed with `pip install black[python2]`. If you want to format +Jupyter Notebooks, install with `pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/pyproject.toml b/pyproject.toml index 79060fc422d..d085c0ddc62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,4 +31,5 @@ build-backend = "setuptools.build_meta" optional-tests = [ "no_python2: run when `python2` extra NOT installed", "no_blackd: run when `d` extra NOT installed", + "no_jupyter: run when `jupyter` extra NOT installed", ] diff --git a/setup.py b/setup.py index 0a00638108c..92b78f1abe1 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def get_long_description() -> str: "regex>=2020.1.8", "pathspec>=0.8.1, <1", "dataclasses>=0.6; python_version < '3.7'", - "typing_extensions>=3.7.4; python_version < '3.8'", + "typing_extensions>=3.10.0.0; python_version < '3.10'", "mypy_extensions>=0.4.3", ], extras_require={ @@ -87,6 +87,7 @@ def get_long_description() -> str: "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], "uvloop": ["uvloop>=0.15.2"], + "jupyter": ["ipython>=7.8.0", "tokenize-rt>=3.2.0"], }, test_suite="tests.test_black", classifiers=[ diff --git a/src/black/__init__.py b/src/black/__init__.py index 51384fb08da..29fb244f8b7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,4 +1,6 @@ import asyncio +from json.decoder import JSONDecodeError +import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor from contextlib import contextmanager from datetime import datetime @@ -18,6 +20,7 @@ Generator, Iterator, List, + MutableMapping, Optional, Pattern, Set, @@ -39,13 +42,21 @@ from black.mode import Feature, supports_feature, VERSION_TO_FEATURES from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache from black.concurrency import cancel, shutdown, maybe_install_uvloop -from black.output import dump_to_file, diff, color_diff, out, err -from black.report import Report, Changed +from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err +from black.report import Report, Changed, NothingChanged from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore from black.files import wrap_stream_for_windows from black.parsing import InvalidInput # noqa F401 from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.handle_ipynb_magics import ( + mask_cell, + unmask_cell, + remove_trailing_semicolon, + put_trailing_semicolon_back, + TRANSFORMED_MAGICS, + jupyter_dependencies_are_installed, +) # lib2to3 fork @@ -60,10 +71,6 @@ NewLine = str -class NothingChanged(UserWarning): - """Raised when reformatted code is the same as source.""" - - class WriteBack(Enum): NO = 0 YES = 1 @@ -196,6 +203,14 @@ def validate_regex( " when piping source on standard input)." ), ) +@click.option( + "--ipynb", + is_flag=True, + help=( + "Format all input files like Jupyter Notebooks regardless of file extension " + "(useful when piping source on standard input)." + ), +) @click.option( "-S", "--skip-string-normalization", @@ -355,6 +370,7 @@ def main( color: bool, fast: bool, pyi: bool, + ipynb: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, experimental_string_processing: bool, @@ -380,6 +396,9 @@ def main( f" the running version `{__version__}`!" ) ctx.exit(1) + if ipynb and pyi: + err("Cannot pass both `pyi` and `ipynb` flags!") + ctx.exit(1) write_back = WriteBack.from_configuration(check=check, diff=diff, color=color) if target_version: @@ -391,6 +410,7 @@ def main( target_versions=versions, line_length=line_length, is_pyi=pyi, + is_ipynb=ipynb, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, experimental_string_processing=experimental_string_processing, @@ -504,6 +524,11 @@ def get_sources( if is_stdin: p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") + if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + verbose=verbose, quiet=quiet + ): + continue + sources.add(p) elif p.is_dir(): sources.update( @@ -516,6 +541,8 @@ def get_sources( force_exclude, report, gitignore, + verbose=verbose, + quiet=quiet, ) ) elif s == "-": @@ -585,6 +612,8 @@ def reformat_one( if is_stdin: if src.suffix == ".pyi": mode = replace(mode, is_pyi=True) + elif src.suffix == ".ipynb": + mode = replace(mode, is_ipynb=True) if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: @@ -733,6 +762,8 @@ def format_file_in_place( """ if src.suffix == ".pyi": mode = replace(mode, is_pyi=True) + elif src.suffix == ".ipynb": + mode = replace(mode, is_ipynb=True) then = datetime.utcfromtimestamp(src.stat().st_mtime) with open(src, "rb") as buf: @@ -741,6 +772,8 @@ def format_file_in_place( dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) except NothingChanged: return False + except JSONDecodeError: + raise ValueError(f"File '{src}' cannot be parsed as valid Jupyter notebook.") if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: @@ -749,7 +782,10 @@ def format_file_in_place( now = datetime.utcnow() src_name = f"{src}\t{then} +0000" dst_name = f"{src}\t{now} +0000" - diff_contents = diff(src_contents, dst_contents, src_name, dst_name) + if mode.is_ipynb: + diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) + else: + diff_contents = diff(src_contents, dst_contents, src_name, dst_name) if write_back == WriteBack.COLOR_DIFF: diff_contents = color_diff(diff_contents) @@ -819,6 +855,29 @@ def format_stdin_to_stdout( f.detach() +def check_stability_and_equivalence( + src_contents: str, dst_contents: str, *, mode: Mode +) -> None: + """Perform stability and equivalence checks. + + Raise AssertionError if source and destination contents are not + equivalent, or if a second pass of the formatter would format the + content differently. + """ + assert_equivalent(src_contents, dst_contents) + + # Forced second pass to work around optional trailing commas (becoming + # forced trailing commas on pass 2) interacting differently with optional + # parentheses. Admittedly ugly. + dst_contents_pass2 = format_str(dst_contents, mode=mode) + if dst_contents != dst_contents_pass2: + dst_contents = dst_contents_pass2 + assert_equivalent(src_contents, dst_contents, pass_num=2) + assert_stable(src_contents, dst_contents, mode=mode) + # Note: no need to explicitly call `assert_stable` if `dst_contents` was + # the same as `dst_contents_pass2`. + + def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Reformat contents of a file and return new contents. @@ -829,26 +888,116 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if not src_contents.strip(): raise NothingChanged - dst_contents = format_str(src_contents, mode=mode) + if mode.is_ipynb: + dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) + else: + dst_contents = format_str(src_contents, mode=mode) if src_contents == dst_contents: raise NothingChanged - if not fast: - assert_equivalent(src_contents, dst_contents) - - # Forced second pass to work around optional trailing commas (becoming - # forced trailing commas on pass 2) interacting differently with optional - # parentheses. Admittedly ugly. - dst_contents_pass2 = format_str(dst_contents, mode=mode) - if dst_contents != dst_contents_pass2: - dst_contents = dst_contents_pass2 - assert_equivalent(src_contents, dst_contents, pass_num=2) - assert_stable(src_contents, dst_contents, mode=mode) - # Note: no need to explicitly call `assert_stable` if `dst_contents` was - # the same as `dst_contents_pass2`. + if not fast and not mode.is_ipynb: + # Jupyter notebooks will already have been checked above. + check_stability_and_equivalence(src_contents, dst_contents, mode=mode) return dst_contents +def validate_cell(src: str) -> None: + """Check that cell does not already contain TransformerManager transformations. + + If a cell contains ``!ls``, then it'll be transformed to + ``get_ipython().system('ls')``. However, if the cell originally contained + ``get_ipython().system('ls')``, then it would get transformed in the same way: + + >>> TransformerManager().transform_cell("get_ipython().system('ls')") + "get_ipython().system('ls')\n" + >>> TransformerManager().transform_cell("!ls") + "get_ipython().system('ls')\n" + + Due to the impossibility of safely roundtripping in such situations, cells + containing transformed magics will be ignored. + """ + if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS): + raise NothingChanged + + +def format_cell(src: str, *, fast: bool, mode: Mode) -> str: + """Format code in given cell of Jupyter notebook. + + General idea is: + + - if cell has trailing semicolon, remove it; + - if cell has IPython magics, mask them; + - format cell; + - reinstate IPython magics; + - reinstate trailing semicolon (if originally present); + - strip trailing newlines. + + Cells with syntax errors will not be processed, as they + could potentially be automagics or multi-line magics, which + are currently not supported. + """ + validate_cell(src) + src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( + src + ) + try: + masked_src, replacements = mask_cell(src_without_trailing_semicolon) + except SyntaxError: + raise NothingChanged + masked_dst = format_str(masked_src, mode=mode) + if not fast: + check_stability_and_equivalence(masked_src, masked_dst, mode=mode) + dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements) + dst = put_trailing_semicolon_back( + dst_without_trailing_semicolon, has_trailing_semicolon + ) + dst = dst.rstrip("\n") + if dst == src: + raise NothingChanged + return dst + + +def validate_metadata(nb: MutableMapping[str, Any]) -> None: + """If notebook is marked as non-Python, don't format it. + + All notebook metadata fields are optional, see + https://nbformat.readthedocs.io/en/latest/format_description.html. So + if a notebook has empty metadata, we will try to parse it anyway. + """ + language = nb.get("metadata", {}).get("language_info", {}).get("name", None) + if language is not None and language != "python": + raise NothingChanged + + +def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: + """Format Jupyter notebook. + + Operate cell-by-cell, only on code cells, only for Python notebooks. + If the ``.ipynb`` originally had a trailing newline, it'll be preseved. + """ + trailing_newline = src_contents[-1] == "\n" + modified = False + nb = json.loads(src_contents) + validate_metadata(nb) + for cell in nb["cells"]: + if cell.get("cell_type", None) == "code": + try: + src = "".join(cell["source"]) + dst = format_cell(src, fast=fast, mode=mode) + except NothingChanged: + pass + else: + cell["source"] = dst.splitlines(keepends=True) + modified = True + if modified: + dst_contents = json.dumps(nb, indent=1, ensure_ascii=False) + if trailing_newline: + dst_contents = dst_contents + "\n" + return dst_contents + else: + raise NothingChanged + + def format_str(src_contents: str, *, mode: Mode) -> FileContent: """Reformat a string and return new contents. diff --git a/src/black/const.py b/src/black/const.py index 821258588ab..dbb4826be0e 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,4 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES = r"\.pyi?$" +DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/files.py b/src/black/files.py index 427ad668f48..5e1ce03d0e6 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -22,6 +22,7 @@ from black.output import err from black.report import Report +from black.handle_ipynb_magics import jupyter_dependencies_are_installed if TYPE_CHECKING: import colorama # noqa: F401 @@ -165,6 +166,9 @@ def gen_python_files( force_exclude: Optional[Pattern[str]], report: Report, gitignore: Optional[PathSpec], + *, + verbose: bool, + quiet: bool, ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, @@ -216,9 +220,15 @@ def gen_python_files( force_exclude, report, gitignore + get_gitignore(child) if gitignore is not None else None, + verbose=verbose, + quiet=quiet, ) elif child.is_file(): + if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + verbose=verbose, quiet=quiet + ): + continue include_match = include.search(normalized_path) if include else True if include_match: yield child diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py new file mode 100644 index 00000000000..ad93c444efc --- /dev/null +++ b/src/black/handle_ipynb_magics.py @@ -0,0 +1,457 @@ +"""Functions to process IPython magics with.""" +from functools import lru_cache +import dataclasses +import ast +from typing import Dict + +import secrets +from typing import List, Tuple +import collections + +from typing import Optional +from typing_extensions import TypeGuard +from black.report import NothingChanged +from black.output import out + + +TRANSFORMED_MAGICS = frozenset( + ( + "get_ipython().run_cell_magic", + "get_ipython().system", + "get_ipython().getoutput", + "get_ipython().run_line_magic", + ) +) +TOKENS_TO_IGNORE = frozenset( + ( + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", + ) +) +NON_PYTHON_CELL_MAGICS = frozenset( + ( + "%%bash", + "%%html", + "%%javascript", + "%%js", + "%%latex", + "%%markdown", + "%%perl", + "%%ruby", + "%%script", + "%%sh", + "%%svg", + "%%writefile", + ) +) + + +@dataclasses.dataclass(frozen=True) +class Replacement: + mask: str + src: str + + +@lru_cache() +def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: + try: + import IPython # noqa:F401 + import tokenize_rt # noqa:F401 + except ModuleNotFoundError: + if verbose or not quiet: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``" + ) + out(msg) + return False + else: + return True + + +def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: + """Remove trailing semicolon from Jupyter notebook cell. + + For example, + + fig, ax = plt.subplots() + ax.plot(x_data, y_data); # plot data + + would become + + fig, ax = plt.subplots() + ax.plot(x_data, y_data) # plot data + + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. + """ + from tokenize_rt import ( + src_to_tokens, + tokens_to_src, + reversed_enumerate, + ) + + tokens = src_to_tokens(src) + trailing_semicolon = False + for idx, token in reversed_enumerate(tokens): + if token.name in TOKENS_TO_IGNORE: + continue + if token.name == "OP" and token.src == ";": + del tokens[idx] + trailing_semicolon = True + break + if not trailing_semicolon: + return src, False + return tokens_to_src(tokens), True + + +def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: + """Put trailing semicolon back if cell originally had it. + + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. + """ + if not has_trailing_semicolon: + return src + from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + + tokens = src_to_tokens(src) + for idx, token in reversed_enumerate(tokens): + if token.name in TOKENS_TO_IGNORE: + continue + tokens[idx] = token._replace(src=token.src + ";") + break + else: # pragma: nocover + raise AssertionError( + "INTERNAL ERROR: Was not able to reinstate trailing semicolon. " + "Please report a bug on https://github.com/psf/black/issues. " + ) from None + return str(tokens_to_src(tokens)) + + +def mask_cell(src: str) -> Tuple[str, List[Replacement]]: + """Mask IPython magics so content becomes parseable Python code. + + For example, + + %matplotlib inline + 'foo' + + becomes + + "25716f358c32750e" + 'foo' + + The replacements are returned, along with the transformed code. + """ + replacements: List[Replacement] = [] + try: + ast.parse(src) + except SyntaxError: + # Might have IPython magics, will process below. + pass + else: + # Syntax is fine, nothing to mask, early return. + return src, replacements + + from IPython.core.inputtransformer2 import TransformerManager + + transformer_manager = TransformerManager() + transformed = transformer_manager.transform_cell(src) + transformed, cell_magic_replacements = replace_cell_magics(transformed) + replacements += cell_magic_replacements + transformed = transformer_manager.transform_cell(transformed) + transformed, magic_replacements = replace_magics(transformed) + if len(transformed.splitlines()) != len(src.splitlines()): + # Multi-line magic, not supported. + raise NothingChanged + replacements += magic_replacements + return transformed, replacements + + +def get_token(src: str, magic: str) -> str: + """Return randomly generated token to mask IPython magic with. + + For example, if 'magic' was `%matplotlib inline`, then a possible + token to mask it with would be `"43fdd17f7e5ddc83"`. The token + will be the same length as the magic, and we make sure that it was + not already present anywhere else in the cell. + """ + assert magic + nbytes = max(len(magic) // 2 - 1, 1) + token = secrets.token_hex(nbytes) + counter = 0 + while token in src: # pragma: nocover + token = secrets.token_hex(nbytes) + counter += 1 + if counter > 100: + raise AssertionError( + "INTERNAL ERROR: Black was not able to replace IPython magic. " + "Please report a bug on https://github.com/psf/black/issues. " + f"The magic might be helpful: {magic}" + ) from None + if len(token) + 2 < len(magic): + token = f"{token}." + return f'"{token}"' + + +def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: + """Replace cell magic with token. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, + + get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\n') + + becomes + + "a794." + ls =!ls + + The replacement, along with the transformed code, is returned. + """ + replacements: List[Replacement] = [] + + tree = ast.parse(src) + + cell_magic_finder = CellMagicFinder() + cell_magic_finder.visit(tree) + if cell_magic_finder.cell_magic is None: + return src, replacements + if cell_magic_finder.cell_magic.header.split()[0] in NON_PYTHON_CELL_MAGICS: + raise NothingChanged + mask = get_token(src, cell_magic_finder.cell_magic.header) + replacements.append(Replacement(mask=mask, src=cell_magic_finder.cell_magic.header)) + return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements + + +def replace_magics(src: str) -> Tuple[str, List[Replacement]]: + """Replace magics within body of cell. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, this + + get_ipython().run_line_magic('matplotlib', 'inline') + 'foo' + + becomes + + "5e67db56d490fd39" + 'foo' + + The replacement, along with the transformed code, are returned. + """ + replacements = [] + magic_finder = MagicFinder() + magic_finder.visit(ast.parse(src)) + new_srcs = [] + for i, line in enumerate(src.splitlines(), start=1): + if i in magic_finder.magics: + offsets_and_magics = magic_finder.magics[i] + if len(offsets_and_magics) != 1: # pragma: nocover + raise AssertionError( + f"Expecting one magic per line, got: {offsets_and_magics}\n" + "Please report a bug on https://github.com/psf/black/issues." + ) + col_offset, magic = ( + offsets_and_magics[0].col_offset, + offsets_and_magics[0].magic, + ) + mask = get_token(src, magic) + replacements.append(Replacement(mask=mask, src=magic)) + line = line[:col_offset] + mask + new_srcs.append(line) + return "\n".join(new_srcs), replacements + + +def unmask_cell(src: str, replacements: List[Replacement]) -> str: + """Remove replacements from cell. + + For example + + "9b20" + foo = bar + + becomes + + %%time + foo = bar + """ + for replacement in replacements: + src = src.replace(replacement.mask, replacement.src) + return src + + +def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: + """Check if attribute is IPython magic. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + """ + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "get_ipython" + ) + + +@dataclasses.dataclass(frozen=True) +class CellMagic: + header: str + body: str + + +@dataclasses.dataclass +class CellMagicFinder(ast.NodeVisitor): + """Find cell magics. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + + For example, + + %%time\nfoo() + + would have been transformed to + + get_ipython().run_cell_magic('time', '', 'foo()\\n') + + and we look for instances of the latter. + """ + + cell_magic: Optional[CellMagic] = None + + def visit_Expr(self, node: ast.Expr) -> None: + """Find cell magic, extract header and body.""" + if ( + isinstance(node.value, ast.Call) + and _is_ipython_magic(node.value.func) + and node.value.func.attr == "run_cell_magic" + ): + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + header = f"%%{args[0]}" + if args[1]: + header += f" {args[1]}" + self.cell_magic = CellMagic(header=header, body=args[2]) + self.generic_visit(node) + + +@dataclasses.dataclass(frozen=True) +class OffsetAndMagic: + col_offset: int + magic: str + + +@dataclasses.dataclass +class MagicFinder(ast.NodeVisitor): + """Visit cell to look for get_ipython calls. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + + For example, + + %matplotlib inline + + would have been transformed to + + get_ipython().run_line_magic('matplotlib', 'inline') + + and we look for instances of the latter (and likewise for other + types of magics). + """ + + magics: Dict[int, List[OffsetAndMagic]] = dataclasses.field( + default_factory=lambda: collections.defaultdict(list) + ) + + def visit_Assign(self, node: ast.Assign) -> None: + """Look for system assign magics. + + For example, + + black_version = !black --version + + would have been transformed to + + black_version = get_ipython().getoutput('black --version') + + and we look for instances of the latter. + """ + if ( + isinstance(node.value, ast.Call) + and _is_ipython_magic(node.value.func) + and node.value.func.attr == "getoutput" + ): + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + assert args + src = f"!{args[0]}" + self.magics[node.value.lineno].append( + OffsetAndMagic(node.value.col_offset, src) + ) + self.generic_visit(node) + + def visit_Expr(self, node: ast.Expr) -> None: + """Look for magics in body of cell. + + For examples, + + !ls + !!ls + ?ls + ??ls + + would (respectively) get transformed to + + get_ipython().system('ls') + get_ipython().getoutput('ls') + get_ipython().run_line_magic('pinfo', 'ls') + get_ipython().run_line_magic('pinfo2', 'ls') + + and we look for instances of any of the latter. + """ + if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + assert args + if node.value.func.attr == "run_line_magic": + if args[0] == "pinfo": + src = f"?{args[1]}" + elif args[0] == "pinfo2": + src = f"??{args[1]}" + else: + src = f"%{args[0]}" + if args[1]: + assert src is not None + src += f" {args[1]}" + elif node.value.func.attr == "system": + src = f"!{args[0]}" + elif node.value.func.attr == "getoutput": + src = f"!!{args[0]}" + else: + raise NothingChanged # unsupported magic. + self.magics[node.value.lineno].append( + OffsetAndMagic(node.value.col_offset, src) + ) + self.generic_visit(node) diff --git a/src/black/mode.py b/src/black/mode.py index e2ce322da5c..0b7624eaf8a 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -101,6 +101,7 @@ class Mode: line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True is_pyi: bool = False + is_ipynb: bool = False magic_trailing_comma: bool = True experimental_string_processing: bool = False @@ -117,6 +118,7 @@ def get_cache_key(self) -> str: str(self.line_length), str(int(self.string_normalization)), str(int(self.is_pyi)), + str(int(self.is_ipynb)), str(int(self.magic_trailing_comma)), str(int(self.experimental_string_processing)), ] diff --git a/src/black/output.py b/src/black/output.py index c253c85e90e..fd3dbb37627 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -3,6 +3,7 @@ The double calls are for patching purposes in tests. """ +import json from typing import Any, Optional from mypy_extensions import mypyc_attr import tempfile @@ -34,6 +35,23 @@ def err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: _err(message, nl=nl, **styles) +def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: + """Return a unified diff string between each cell in notebooks `a` and `b`.""" + a_nb = json.loads(a) + b_nb = json.loads(b) + diff_lines = [ + diff( + "".join(a_nb["cells"][cell_number]["source"]) + "\n", + "".join(b_nb["cells"][cell_number]["source"]) + "\n", + f"{a_name}:cell_{cell_number}", + f"{b_name}:cell_{cell_number}", + ) + for cell_number, cell in enumerate(a_nb["cells"]) + if cell["cell_type"] == "code" + ] + return "".join(diff_lines) + + def diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between strings `a` and `b`.""" import difflib diff --git a/src/black/report.py b/src/black/report.py index 8fc5da2e167..7e1c8b4b87f 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -16,6 +16,10 @@ class Changed(Enum): YES = 2 +class NothingChanged(UserWarning): + """Raised when reformatted code is the same as source.""" + + @dataclass class Report: """Provides a reformatting counter. Can be rendered with `str(report)`.""" diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 90643987942..b37f8bf6516 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -111,6 +111,13 @@ "long_checkout": false, "py_versions": ["all"] }, + "scikit-lego": { + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/koaning/scikit-lego", + "long_checkout": false, + "py_versions": ["all"] + }, "sqlalchemy": { "no_cli_args_reason": "breaks black with new string parsing - #2188", "cli_arguments": [], diff --git a/tests/data/non_python_notebook.ipynb b/tests/data/non_python_notebook.ipynb new file mode 100644 index 00000000000..da5cdd8e185 --- /dev/null +++ b/tests/data/non_python_notebook.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(tidyverse) ","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle"},"execution_count":null,"outputs":[]}]} \ No newline at end of file diff --git a/tests/data/notebook_empty_metadata.ipynb b/tests/data/notebook_empty_metadata.ipynb new file mode 100644 index 00000000000..7dc1f805cd6 --- /dev/null +++ b/tests/data/notebook_empty_metadata.ipynb @@ -0,0 +1,27 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/data/notebook_no_trailing_newline.ipynb b/tests/data/notebook_no_trailing_newline.ipynb new file mode 100644 index 00000000000..79f95bea2f6 --- /dev/null +++ b/tests/data/notebook_no_trailing_newline.ipynb @@ -0,0 +1,39 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/data/notebook_trailing_newline.ipynb b/tests/data/notebook_trailing_newline.ipynb new file mode 100644 index 00000000000..4f82869312d --- /dev/null +++ b/tests/data/notebook_trailing_newline.ipynb @@ -0,0 +1,39 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/data/notebook_which_cant_be_parsed.ipynb b/tests/data/notebook_which_cant_be_parsed.ipynb new file mode 100644 index 00000000000..257cc5642cb --- /dev/null +++ b/tests/data/notebook_which_cant_be_parsed.ipynb @@ -0,0 +1 @@ +foo diff --git a/tests/data/notebook_without_changes.ipynb b/tests/data/notebook_without_changes.ipynb new file mode 100644 index 00000000000..ac6c7e63efa --- /dev/null +++ b/tests/data/notebook_without_changes.ipynb @@ -0,0 +1,46 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"foo\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook should not be reformatted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index e3be9c71fd4..0a716d764a8 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1378,6 +1378,8 @@ def test_include_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1689,6 +1691,8 @@ def test_gitignore_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1716,6 +1720,8 @@ def test_nested_gitignore(self) -> None: None, report, root_gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1750,6 +1756,8 @@ def test_empty_include(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1774,6 +1782,8 @@ def test_extend_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1846,6 +1856,8 @@ def test_symlink_out_of_root_directory(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) except ValueError as ve: @@ -1867,6 +1879,8 @@ def test_symlink_out_of_root_directory(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) path.iterdir.assert_called() diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py new file mode 100644 index 00000000000..038155e9270 --- /dev/null +++ b/tests/test_ipynb.py @@ -0,0 +1,455 @@ +import pathlib +from click.testing import CliRunner +from black.handle_ipynb_magics import jupyter_dependencies_are_installed +from black import ( + main, + NothingChanged, + format_cell, + format_file_contents, + format_file_in_place, +) +import os +import pytest +from black import Mode +from _pytest.monkeypatch import MonkeyPatch +from py.path import local + +pytestmark = pytest.mark.jupyter +pytest.importorskip("IPython", reason="IPython is an optional dependency") +pytest.importorskip("tokenize_rt", reason="tokenize-rt is an optional dependency") + +JUPYTER_MODE = Mode(is_ipynb=True) + +runner = CliRunner() + + +def test_noop() -> None: + src = 'foo = "a"' + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +@pytest.mark.parametrize("fast", [True, False]) +def test_trailing_semicolon(fast: bool) -> None: + src = 'foo = "a" ;' + result = format_cell(src, fast=fast, mode=JUPYTER_MODE) + expected = 'foo = "a";' + assert result == expected + + +def test_trailing_semicolon_with_comment() -> None: + src = 'foo = "a" ; # bar' + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + expected = 'foo = "a"; # bar' + assert result == expected + + +def test_trailing_semicolon_with_comment_on_next_line() -> None: + src = "import black;\n\n# this is a comment" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_trailing_semicolon_indented() -> None: + src = "with foo:\n plot_bar();" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_trailing_semicolon_noop() -> None: + src = 'foo = "a";' + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_cell_magic() -> None: + src = "%%time\nfoo =bar" + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + expected = "%%time\nfoo = bar" + assert result == expected + + +def test_cell_magic_noop() -> None: + src = "%%time\n2 + 2" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +@pytest.mark.parametrize( + "src, expected", + ( + pytest.param("ls =!ls", "ls = !ls", id="System assignment"), + pytest.param("!ls\n'foo'", '!ls\n"foo"', id="System call"), + pytest.param("!!ls\n'foo'", '!!ls\n"foo"', id="Other system call"), + pytest.param("?str\n'foo'", '?str\n"foo"', id="Help"), + pytest.param("??str\n'foo'", '??str\n"foo"', id="Other help"), + pytest.param( + "%matplotlib inline\n'foo'", + '%matplotlib inline\n"foo"', + id="Line magic with argument", + ), + pytest.param("%time\n'foo'", '%time\n"foo"', id="Line magic without argument"), + ), +) +def test_magic(src: str, expected: str) -> None: + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + assert result == expected + + +@pytest.mark.parametrize( + "src", + ( + "%%bash\n2+2", + "%%html --isolated\n2+2", + ), +) +def test_non_python_magics(src: str) -> None: + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_set_input() -> None: + src = "a = b??" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_input_already_contains_transformed_magic() -> None: + src = '%time foo()\nget_ipython().run_cell_magic("time", "", "foo()\\n")' + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_magic_noop() -> None: + src = "ls = !ls" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_cell_magic_with_magic() -> None: + src = "%%t -n1\nls =!ls" + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + expected = "%%t -n1\nls = !ls" + assert result == expected + + +def test_cell_magic_nested() -> None: + src = "%%time\n%%time\n2+2" + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + expected = "%%time\n%%time\n2 + 2" + assert result == expected + + +def test_cell_magic_with_magic_noop() -> None: + src = "%%t -n1\nls = !ls" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_automagic() -> None: + src = "pip install black" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_multiline_magic() -> None: + src = "%time 1 + \\\n2" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_multiline_no_magic() -> None: + src = "1 + \\\n2" + result = format_cell(src, fast=True, mode=JUPYTER_MODE) + expected = "1 + 2" + assert result == expected + + +def test_cell_magic_with_invalid_body() -> None: + src = "%%time\nif True" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_empty_cell() -> None: + src = "" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + +def test_entire_notebook_empty_metadata() -> None: + with open( + os.path.join("tests", "data", "notebook_empty_metadata.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {},\n' + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}\n" + ) + assert result == expected + + +def test_entire_notebook_trailing_newline() -> None: + with open( + os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {\n' + ' "interpreter": {\n' + ' "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8"\n' # noqa:B950 + " },\n" + ' "kernelspec": {\n' + ' "display_name": "Python 3.8.10 64-bit (\'black\': venv)",\n' + ' "name": "python3"\n' + " },\n" + ' "language_info": {\n' + ' "name": "python",\n' + ' "version": ""\n' + " }\n" + " },\n" + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}\n" + ) + assert result == expected + + +def test_entire_notebook_no_trailing_newline() -> None: + with open( + os.path.join("tests", "data", "notebook_no_trailing_newline.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {\n' + ' "interpreter": {\n' + ' "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8"\n' # noqa: B950 + " },\n" + ' "kernelspec": {\n' + ' "display_name": "Python 3.8.10 64-bit (\'black\': venv)",\n' + ' "name": "python3"\n' + " },\n" + ' "language_info": {\n' + ' "name": "python",\n' + ' "version": ""\n' + " }\n" + " },\n" + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}" + ) + assert result == expected + + +def test_entire_notebook_without_changes() -> None: + with open( + os.path.join("tests", "data", "notebook_without_changes.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + with pytest.raises(NothingChanged): + format_file_contents(content, fast=True, mode=JUPYTER_MODE) + + +def test_non_python_notebook() -> None: + with open(os.path.join("tests", "data", "non_python_notebook.ipynb"), "rb") as fd: + content_bytes = fd.read() + content = content_bytes.decode() + with pytest.raises(NothingChanged): + format_file_contents(content, fast=True, mode=JUPYTER_MODE) + + +def test_empty_string() -> None: + with pytest.raises(NothingChanged): + format_file_contents("", fast=True, mode=JUPYTER_MODE) + + +def test_unparseable_notebook() -> None: + msg = ( + r"File 'tests[/\\]data[/\\]notebook_which_cant_be_parsed\.ipynb' " + r"cannot be parsed as valid Jupyter notebook\." + ) + with pytest.raises(ValueError, match=msg): + format_file_in_place( + pathlib.Path("tests") / "data/notebook_which_cant_be_parsed.ipynb", + fast=True, + mode=JUPYTER_MODE, + ) + + +def test_ipynb_diff_with_change() -> None: + result = runner.invoke( + main, + [ + os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), + "--diff", + ], + ) + expected = "@@ -1,3 +1,3 @@\n %%time\n \n-print('foo')\n" '+print("foo")\n' + assert expected in result.output + + +def test_ipynb_diff_with_no_change() -> None: + result = runner.invoke( + main, + [ + os.path.join("tests", "data", "notebook_without_changes.ipynb"), + "--diff", + ], + ) + expected = "1 file would be left unchanged." + assert expected in result.output + + +def test_cache_isnt_written_if_no_jupyter_deps_single( + monkeypatch: MonkeyPatch, tmpdir: local +) -> None: + # Check that the cache isn't written to if Jupyter dependencies aren't installed. + jupyter_dependencies_are_installed.cache_clear() + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False + ) + result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + assert "No Python files are present to be formatted. Nothing to do" in result.output + jupyter_dependencies_are_installed.cache_clear() + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True + ) + result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + assert "reformatted" in result.output + + +def test_cache_isnt_written_if_no_jupyter_deps_dir( + monkeypatch: MonkeyPatch, tmpdir: local +) -> None: + # Check that the cache isn't written to if Jupyter dependencies aren't installed. + jupyter_dependencies_are_installed.cache_clear() + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + monkeypatch.setattr( + "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False + ) + result = runner.invoke(main, [str(tmpdir)]) + assert "No Python files are present to be formatted. Nothing to do" in result.output + jupyter_dependencies_are_installed.cache_clear() + monkeypatch.setattr( + "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: True + ) + result = runner.invoke(main, [str(tmpdir)]) + assert "reformatted" in result.output + + +def test_ipynb_flag(tmpdir: local) -> None: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.a_file_extension_which_is_definitely_not_ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + result = runner.invoke( + main, + [ + str(tmp_nb), + "--diff", + "--ipynb", + ], + ) + expected = "@@ -1,3 +1,3 @@\n %%time\n \n-print('foo')\n" '+print("foo")\n' + assert expected in result.output + + +def test_ipynb_and_pyi_flags() -> None: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + result = runner.invoke( + main, + [ + nb, + "--pyi", + "--ipynb", + "--diff", + ], + ) + assert isinstance(result.exception, SystemExit) + expected = "Cannot pass both `pyi` and `ipynb` flags!\n" + assert result.output == expected diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py new file mode 100644 index 00000000000..bcda2d5369f --- /dev/null +++ b/tests/test_no_ipynb.py @@ -0,0 +1,37 @@ +import pytest +import os + +from tests.util import THIS_DIR +from black import main, jupyter_dependencies_are_installed +from click.testing import CliRunner +from _pytest.tmpdir import tmpdir + +pytestmark = pytest.mark.no_jupyter + +runner = CliRunner() + + +def test_ipynb_diff_with_no_change_single() -> None: + jupyter_dependencies_are_installed.cache_clear() + path = THIS_DIR / "data/notebook_trailing_newline.ipynb" + result = runner.invoke(main, [str(path)]) + expected_output = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``\n" + ) + assert expected_output in result.output + + +def test_ipynb_diff_with_no_change_dir(tmpdir: tmpdir) -> None: + jupyter_dependencies_are_installed.cache_clear() + runner = CliRunner() + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + result = runner.invoke(main, [str(tmpdir)]) + expected_output = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``\n" + ) + assert expected_output in result.output diff --git a/tox.ini b/tox.ini index a97d18946b3..8d4fabaf520 100644 --- a/tox.ini +++ b/tox.ini @@ -16,10 +16,17 @@ commands = pip install -e .[d] coverage erase pytest tests --run-optional no_python2 \ + --run-optional no_jupyter \ !ci: --numprocesses auto \ --cov {posargs} pip install -e .[d,python2] pytest tests --run-optional python2 \ + --run-optional no_jupyter \ + !ci: --numprocesses auto \ + --cov --cov-append {posargs} + pip install -e .[jupyter] + pytest tests --run-optional jupyter \ + -m jupyter \ !ci: --numprocesses auto \ --cov --cov-append {posargs} coverage report