From 05fee793c7ccff4fd91426284d61e7f491d69fcf Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Fri, 2 Jul 2021 10:09:21 +0100 Subject: [PATCH 01/81] wip --- .gitignore | 1 + setup.py | 1 + src/black/__init__.py | 78 ++++++++- src/black/const.py | 2 +- src/black/handle_ipynb_magics.py | 261 +++++++++++++++++++++++++++++++ src/black/mode.py | 2 + tests/test_ipynb.py | 99 ++++++++++++ 7 files changed, 442 insertions(+), 2 deletions(-) create mode 100644 src/black/handle_ipynb_magics.py create mode 100644 tests/test_ipynb.py diff --git a/.gitignore b/.gitignore index ab796ce4cd0..f81bce8fd4e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ src/_black_version.py *.swp .hypothesis/ venv/ +.ipynb_checkpoints/ diff --git a/setup.py b/setup.py index 5549ae35342..4e024fdc2ec 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ def get_long_description() -> str: "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], "uvloop": ["uvloop>=0.15.2"], + "jupyter": ["ipython>=7.8.0", "tokenize-rt>=3.2.0"], }, test_suite="tests.test_black", classifiers=[ diff --git a/src/black/__init__.py b/src/black/__init__.py index 8e2123d50cc..39468c77973 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,4 +1,5 @@ import asyncio +import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor from contextlib import contextmanager from datetime import datetime @@ -46,6 +47,12 @@ from black.files import wrap_stream_for_windows from black.parsing import InvalidInput # noqa F401 from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.handle_ipynb_magics import ( + mask_cell, + unmask_cell, + remove_trailing_semicolon, + put_trailing_semicolon_back, +) # lib2to3 fork @@ -196,6 +203,14 @@ def validate_regex( " when piping source on standard input)." ), ) +@click.option( + "--ipynb", + is_flag=True, + help=( + "Format all input files like ipynb notebooks regardless of file extension " + "(useful when piping source on standard input)." + ), +) @click.option( "-S", "--skip-string-normalization", @@ -354,6 +369,7 @@ def main( color: bool, fast: bool, pyi: bool, + ipynb: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, experimental_string_processing: bool, @@ -390,6 +406,7 @@ def main( target_versions=versions, line_length=line_length, is_pyi=pyi, + is_ipynb=ipynb, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, experimental_string_processing=experimental_string_processing, @@ -584,6 +601,8 @@ def reformat_one( if is_stdin: if src.suffix == ".pyi": mode = replace(mode, is_pyi=True) + elif src.suffix == ".ipynb": + mode = replace(mode, is_ipynb=True) if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: @@ -731,7 +750,15 @@ def format_file_in_place( `mode` and `fast` options are passed to :func:`format_file_contents`. """ if src.suffix == ".pyi": - mode = replace(mode, is_pyi=True) + mode = replace(mode, is_pyi=True, is_ipynb=False) + elif src.suffix == ".ipynb": + mode = replace( + mode, + is_pyi=False, + is_ipynb=True, + ) + elif src.suffix == ".py": + mode = replace(mode, is_pyi=False, is_ipynb=False) then = datetime.utcfromtimestamp(src.stat().st_mtime) with open(src, "rb") as buf: @@ -825,6 +852,9 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. `mode` is passed to :func:`format_str`. """ + if mode.is_ipynb: + return format_ipynb_string(src_contents, mode=mode, fast=fast) + if not src_contents.strip(): raise NothingChanged @@ -848,6 +878,52 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo return dst_contents +def format_cell(src: str, *, mode: Mode) -> str: + src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( + src + ) + try: + masked_cell, replacements = mask_cell(src_without_trailing_semicolon) + except SyntaxError: + # Don't format, might be automagic or multi-line magic. + raise NothingChanged + formatted_masked_cell = format_str(masked_cell, mode=mode) + formatted_cell = unmask_cell(formatted_masked_cell, replacements) + new_src = put_trailing_semicolon_back(formatted_cell, has_trailing_semicolon) + new_src = new_src.rstrip("\n") + if new_src == src: + raise NothingChanged + return new_src + + +def format_ipynb_string( + src_contents: str, *, mode: Mode, fast: bool = False +) -> FileContent: + nb = json.loads(src_contents) + trailing_newline = src_contents[-1] == "\n" + modified = False + for _, cell in enumerate(nb["cells"]): + if cell.get("cell_type", None) == "code": + try: + src = "".join(cell["source"]) + new_src = format_cell(src, mode=mode) + except NothingChanged: + pass + else: + cell["source"] = new_src.splitlines(keepends=True) + modified = True + + if modified: + res = json.dumps(nb, indent=1, ensure_ascii=False) + if trailing_newline: + res = res + "\n" + if res == src_contents: + raise NothingChanged + return res + else: + raise NothingChanged + + def format_str(src_contents: str, *, mode: Mode) -> FileContent: """Reformat a string and return new contents. diff --git a/src/black/const.py b/src/black/const.py index 821258588ab..dbb4826be0e 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,4 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES = r"\.pyi?$" +DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py new file mode 100644 index 00000000000..5a713a91a84 --- /dev/null +++ b/src/black/handle_ipynb_magics.py @@ -0,0 +1,261 @@ +import ast +from typing import Dict + +import secrets +from tokenize_rt import ( + src_to_tokens, + tokens_to_src, + NON_CODING_TOKENS, + reversed_enumerate, +) +from typing import NamedTuple, List, Tuple +import collections + +from typing import Optional + + +class Replacement(NamedTuple): + mask: str + src: str + + +class UnsupportedMagic(UserWarning): + """Raise when Magic (e.g. `a = b??`) is not supported.""" + + +def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: + tokens = src_to_tokens(src) + trailing_semicolon = False + for idx, token in reversed_enumerate(tokens): + if token.name in NON_CODING_TOKENS or token.name == "NEWLINE" or not token.src: + continue + if token.name == "OP" and token.src == ";": + del tokens[idx] + trailing_semicolon = True + break + if not trailing_semicolon: + return src, False + return tokens_to_src(tokens), True + + +def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: + if not has_trailing_semicolon: + return src + tokens = src_to_tokens(src) + for idx, token in reversed_enumerate(tokens): + if token.name in NON_CODING_TOKENS or token.name == "NEWLINE" or not token.src: + continue + tokens[idx] = token._replace(src=token.src + ";") + break + else: # pragma: nocover + raise AssertionError("Unreachable code") + return str(tokens_to_src(tokens)) + + +def mask_cell(src: str) -> Tuple[str, List[Replacement]]: + replacements: List[Replacement] = [] + try: + ast.parse(src) + except SyntaxError: + # Might be able to parse it with IPython + pass + else: + # Syntax is fine, nothing to mask + return src, replacements + + from IPython.core.inputtransformer2 import TransformerManager + + transformer_manager = TransformerManager() + transformed = transformer_manager.transform_cell(src) + + transformed, cell_magic_replacements = replace_cell_magics(transformed) + replacements += cell_magic_replacements + + transformed = transformer_manager.transform_cell(transformed) + try: + transformed, magic_replacements = replace_magics(transformed) + except UnsupportedMagic: + # will be ignored upstream + raise SyntaxError + + replacements += magic_replacements + + return transformed, replacements + + +def get_token(src: str, *, is_cell_magic: bool = False) -> str: + token = secrets.token_hex(3) + while token in src: # pragma: nocover + token = secrets.token_hex(3) + if is_cell_magic: + return f"# {token}" + return f'str("{token}")' + + +def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: + replacements: List[Replacement] = [] + + tree = ast.parse(src) + + cell_magic_finder = CellMagicFinder() + cell_magic_finder.visit(tree) + if not cell_magic_finder.header: + return src, replacements + mask = get_token(src, is_cell_magic=True) + replacements.append(Replacement(mask=mask, src=cell_magic_finder.header)) + return f"{mask}\n{cell_magic_finder.body}", replacements + + +def replace_magics(src: str) -> Tuple[str, List[Replacement]]: + replacements = [] + + tree = ast.parse(src) + + magic_finder = MagicFinder() + magic_finder.visit(tree) + new_srcs = [] + for i, line in enumerate(src.splitlines(), start=1): + if i in magic_finder.magics: + magics = magic_finder.magics[i] + if len(magics) != 1: + raise UnsupportedMagic + col_offset, magic = magic_finder.magics[i][0] + mask = get_token(src) + replacements.append(Replacement(mask=mask, src=magic)) + line = line[:col_offset] + mask + new_srcs.append(line) + return "\n".join(new_srcs), replacements + + +def unmask_cell(src: str, replacements: List[Replacement]) -> str: + for replacement in replacements: + src = src.replace(replacement.mask, replacement.src) + return src + + +def _is_ipython_magic(node: ast.expr) -> bool: + """Check if attribute is IPython magic.""" + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id == "get_ipython" + ) + + +class CellMagicFinder(ast.NodeVisitor): + """Find cell magics.""" + + def __init__(self) -> None: + """Record where cell magics occur.""" + self.header: Optional[str] = None + self.body: Optional[str] = None + + def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103 + """ + Find cell magic, extract header and body. + Raises + ------ + AssertionError + Defensive check. + """ + if ( + isinstance(node.value, ast.Call) + and _is_ipython_magic(node.value.func) + and isinstance(node.value.func, ast.Attribute) + and node.value.func.attr == "run_cell_magic" + ): + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + header: Optional[str] = f"%%{args[0]}" + if args[1]: + assert header is not None + header += f" {args[1]}" + self.header = header + self.body = args[2] + self.generic_visit(node) + + +class MagicFinder(ast.NodeVisitor): + """Visit cell to look for get_ipython calls.""" + + def __init__(self) -> None: + """Magics will record where magics occur.""" + self.magics: Dict[int, List[Tuple[int, str]]] = collections.defaultdict(list) + + def visit_Assign(self, node: ast.Assign) -> None: # pylint: disable=C0103,R0912 + """ + Get source to replace ipython magic with. + Parameters + ---------- + node + Function call. + Raises + ------ + AssertionError + Defensive check. + """ + if ( + isinstance(node.value, ast.Call) + and _is_ipython_magic(node.value.func) + and isinstance(node.value.func, ast.Attribute) + and node.value.func.attr == "getoutput" + ): + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + assert args + src = f"!{args[0]}" + self.magics[node.value.lineno].append( + ( + node.value.col_offset, + src, + ) + ) + self.generic_visit(node) + + def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103,R0912 + """ + Get source to replace ipython magic with. + Parameters + ---------- + node + Function call. + Raises + ------ + AssertionError + Defensive check. + """ + if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): + assert isinstance(node.value.func, ast.Attribute) # help mypy + args = [] + for arg in node.value.args: + assert isinstance(arg, ast.Str) + args.append(arg.s) + assert args + if node.value.func.attr == "run_line_magic": + if args[0] == "pinfo": + src = f"?{args[1]}" + elif args[0] == "pinfo2": + src = f"??{args[1]}" + else: + src = f"%{args[0]}" + if args[1]: + assert src is not None + src += f" {args[1]}" + elif node.value.func.attr == "system": + src = f"!{args[0]}" + elif node.value.func.attr == "getoutput": + src = f"!!{args[0]}" + else: + raise UnsupportedMagic + self.magics[node.value.lineno].append( + ( + node.value.col_offset, + src, + ) + ) + self.generic_visit(node) diff --git a/src/black/mode.py b/src/black/mode.py index e2ce322da5c..0b7624eaf8a 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -101,6 +101,7 @@ class Mode: line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True is_pyi: bool = False + is_ipynb: bool = False magic_trailing_comma: bool = True experimental_string_processing: bool = False @@ -117,6 +118,7 @@ def get_cache_key(self) -> str: str(self.line_length), str(int(self.string_normalization)), str(int(self.is_pyi)), + str(int(self.is_ipynb)), str(int(self.magic_trailing_comma)), str(int(self.experimental_string_processing)), ] diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py new file mode 100644 index 00000000000..a41c76a6a60 --- /dev/null +++ b/tests/test_ipynb.py @@ -0,0 +1,99 @@ +from black import NothingChanged, format_cell +from tests.util import DEFAULT_MODE +import pytest + + +def test_noop() -> None: + src = 'foo = "a"' + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_trailing_semicolon() -> None: + src = 'foo = "a" ;' + result = format_cell(src, mode=DEFAULT_MODE) + expected = 'foo = "a";' + assert result == expected + + +def test_trailing_semicolon_noop() -> None: + src = 'foo = "a";' + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_cell_magic() -> None: + src = "%%time\nfoo =bar" + result = format_cell(src, mode=DEFAULT_MODE) + expected = "%%time\nfoo = bar" + assert result == expected + + +def test_cell_magic_noop() -> None: + src = "%%time\n2 + 2" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +@pytest.mark.parametrize( + "src, expected", + ( + pytest.param("ls =!ls", "ls = !ls", id="System assignment"), + pytest.param("!ls\n'foo'", '!ls\n"foo"', id="System call"), + pytest.param("!!ls\n'foo'", '!!ls\n"foo"', id="Other system call"), + pytest.param("?str\n'foo'", '?str\n"foo"', id="Help"), + pytest.param("??str\n'foo'", '??str\n"foo"', id="Other help"), + pytest.param( + "%matplotlib inline\n'foo'", + '%matplotlib inline\n"foo"', + id="Line magic with argument", + ), + pytest.param("%time\n'foo'", '%time\n"foo"', id="Line magic without argument"), + ), +) +def test_magic(src: str, expected: str) -> None: + result = format_cell(src, mode=DEFAULT_MODE) + assert result == expected + + +def test_set_input() -> None: + src = "a = b??" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_magic_noop() -> None: + src = "ls = !ls" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_cell_magic_with_magic() -> None: + src = "%%t -n1\nls =!ls" + result = format_cell(src, mode=DEFAULT_MODE) + expected = "%%t -n1\nls = !ls" + assert result == expected + + +def test_cell_magic_with_magic_noop() -> None: + src = "%%t -n1\nls = !ls" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_automagic() -> None: + src = "pip install black" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_cell_magic_with_invalid_body() -> None: + src = "%%time\nif True" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_empty_cell() -> None: + src = "" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) From 55bdaed6754e927c7d1293443d05529adeeafad5 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 3 Jul 2021 21:59:07 +0100 Subject: [PATCH 02/81] fixup tests --- src/black/__init__.py | 2 +- src/black/handle_ipynb_magics.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 39468c77973..cece9776f11 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -758,7 +758,7 @@ def format_file_in_place( is_ipynb=True, ) elif src.suffix == ".py": - mode = replace(mode, is_pyi=False, is_ipynb=False) + mode = replace(mode, is_ipynb=False) then = datetime.utcfromtimestamp(src.stat().st_mtime) with open(src, "rb") as buf: diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 5a713a91a84..dc0ca516c97 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -2,12 +2,6 @@ from typing import Dict import secrets -from tokenize_rt import ( - src_to_tokens, - tokens_to_src, - NON_CODING_TOKENS, - reversed_enumerate, -) from typing import NamedTuple, List, Tuple import collections @@ -24,6 +18,13 @@ class UnsupportedMagic(UserWarning): def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: + from tokenize_rt import ( + src_to_tokens, + tokens_to_src, + NON_CODING_TOKENS, + reversed_enumerate, + ) + tokens = src_to_tokens(src) trailing_semicolon = False for idx, token in reversed_enumerate(tokens): @@ -39,6 +40,13 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: + from tokenize_rt import ( + src_to_tokens, + tokens_to_src, + NON_CODING_TOKENS, + reversed_enumerate, + ) + if not has_trailing_semicolon: return src tokens = src_to_tokens(src) From 61bb0150d4d8f14971d22f79d93b5fe103dfe4b5 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 3 Jul 2021 22:09:16 +0100 Subject: [PATCH 03/81] skip tests if no IPython --- .github/workflows/ipynb_test.yml | 45 ++++++++++++++++++++++++++++++++ tests/test_ipynb.py | 2 ++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/ipynb_test.yml diff --git a/.github/workflows/ipynb_test.yml b/.github/workflows/ipynb_test.yml new file mode 100644 index 00000000000..eb09b7c756d --- /dev/null +++ b/.github/workflows/ipynb_test.yml @@ -0,0 +1,45 @@ +name: test ipynb + +on: + push: + paths-ignore: + - "docs/**" + - "*.md" + + pull_request: + paths-ignore: + - "docs/**" + - "*.md" + +jobs: + build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. Without this if check, checks are duplicated since + # internal PRs match both the push and pull_request events. + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + + - name: Install latest pip + run: | + python -m pip install --upgrade pip + + - name: Test Jupyter Extra Install + run: | + python -m pip install -e ".[jupyter]" + + - name: Primer uvloop run + run: | + pytest tests/test_ipynb.py diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index a41c76a6a60..922d83ba865 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -2,6 +2,8 @@ from tests.util import DEFAULT_MODE import pytest +pytest.importorskip("IPython") + def test_noop() -> None: src = 'foo = "a"' From 53cc3f474a3dd3b674387b686de4170716bd8381 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 09:15:30 +0100 Subject: [PATCH 04/81] install test requirements in ipynb tests --- .github/workflows/ipynb_test.yml | 5 +++-- src/black/handle_ipynb_magics.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ipynb_test.yml b/.github/workflows/ipynb_test.yml index eb09b7c756d..0be99eb184e 100644 --- a/.github/workflows/ipynb_test.yml +++ b/.github/workflows/ipynb_test.yml @@ -39,7 +39,8 @@ jobs: - name: Test Jupyter Extra Install run: | python -m pip install -e ".[jupyter]" + python -m pip install -r test_requirements.txt - - name: Primer uvloop run + - name: Run Jupyter tests run: | - pytest tests/test_ipynb.py + pytest tests/test_ipynb.py --cov=black.handle_ipynb_magics --cov-report=term-missing --cov-branch diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index dc0ca516c97..c7b678b5c8d 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -125,7 +125,8 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: for i, line in enumerate(src.splitlines(), start=1): if i in magic_finder.magics: magics = magic_finder.magics[i] - if len(magics) != 1: + if len(magics) != 1: # pragma: nocover + # defensive check raise UnsupportedMagic col_offset, magic = magic_finder.magics[i][0] mask = get_token(src) From 27aa4dc6ce271b165115564094cdf02c8b4a2e1c Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 09:19:02 +0100 Subject: [PATCH 05/81] if --ipynb format all as ipynb --- src/black/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index cece9776f11..f442ff85002 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -750,15 +750,9 @@ def format_file_in_place( `mode` and `fast` options are passed to :func:`format_file_contents`. """ if src.suffix == ".pyi": - mode = replace(mode, is_pyi=True, is_ipynb=False) + mode = replace(mode, is_pyi=True) elif src.suffix == ".ipynb": - mode = replace( - mode, - is_pyi=False, - is_ipynb=True, - ) - elif src.suffix == ".py": - mode = replace(mode, is_ipynb=False) + mode = replace(mode, is_ipynb=True) then = datetime.utcfromtimestamp(src.stat().st_mtime) with open(src, "rb") as buf: From 33887d208a5607f6db7bd46a06dae9ab7322bd9e Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 09:40:24 +0100 Subject: [PATCH 06/81] wip --- src/black/__init__.py | 2 + tests/data/notebook_no_trailing_newline.ipynb | 39 +++++++++++++++++++ tests/data/notebook_trailing_newline.ipynb | 39 +++++++++++++++++++ tests/test_ipynb.py | 4 ++ 4 files changed, 84 insertions(+) create mode 100644 tests/data/notebook_no_trailing_newline.ipynb create mode 100644 tests/data/notebook_trailing_newline.ipynb diff --git a/src/black/__init__.py b/src/black/__init__.py index f442ff85002..ef839e44312 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -894,6 +894,8 @@ def format_ipynb_string( src_contents: str, *, mode: Mode, fast: bool = False ) -> FileContent: nb = json.loads(src_contents) + if not src_contents: + raise NothingChanged trailing_newline = src_contents[-1] == "\n" modified = False for _, cell in enumerate(nb["cells"]): diff --git a/tests/data/notebook_no_trailing_newline.ipynb b/tests/data/notebook_no_trailing_newline.ipynb new file mode 100644 index 00000000000..79f95bea2f6 --- /dev/null +++ b/tests/data/notebook_no_trailing_newline.ipynb @@ -0,0 +1,39 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/data/notebook_trailing_newline.ipynb b/tests/data/notebook_trailing_newline.ipynb new file mode 100644 index 00000000000..4f82869312d --- /dev/null +++ b/tests/data/notebook_trailing_newline.ipynb @@ -0,0 +1,39 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 922d83ba865..6ccff4e8c76 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -99,3 +99,7 @@ def test_empty_cell() -> None: src = "" with pytest.raises(NothingChanged): format_cell(src, mode=DEFAULT_MODE) + + +def test_entire_notebook() -> None: + pass From 9e23bc62e787dbe1bd875d9befef454be8d42c1d Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 10:42:03 +0100 Subject: [PATCH 07/81] add some whole-notebook tests --- src/black/__init__.py | 5 +- tests/data/notebook_without_changes.ipynb | 46 +++++++++ tests/test_ipynb.py | 120 +++++++++++++++++++++- 3 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 tests/data/notebook_without_changes.ipynb diff --git a/src/black/__init__.py b/src/black/__init__.py index ef839e44312..0db4a72e8e7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -893,9 +893,9 @@ def format_cell(src: str, *, mode: Mode) -> str: def format_ipynb_string( src_contents: str, *, mode: Mode, fast: bool = False ) -> FileContent: - nb = json.loads(src_contents) if not src_contents: raise NothingChanged + nb = json.loads(src_contents) trailing_newline = src_contents[-1] == "\n" modified = False for _, cell in enumerate(nb["cells"]): @@ -913,7 +913,8 @@ def format_ipynb_string( res = json.dumps(nb, indent=1, ensure_ascii=False) if trailing_newline: res = res + "\n" - if res == src_contents: + if res == src_contents: # pragma: nocover + # Defensive check raise NothingChanged return res else: diff --git a/tests/data/notebook_without_changes.ipynb b/tests/data/notebook_without_changes.ipynb new file mode 100644 index 00000000000..ac6c7e63efa --- /dev/null +++ b/tests/data/notebook_without_changes.ipynb @@ -0,0 +1,46 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"foo\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook should not be reformatted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('black': venv)", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 6ccff4e8c76..9fd4108e559 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,4 +1,5 @@ -from black import NothingChanged, format_cell +from black import NothingChanged, format_cell, format_ipynb_string +import os from tests.util import DEFAULT_MODE import pytest @@ -101,5 +102,118 @@ def test_empty_cell() -> None: format_cell(src, mode=DEFAULT_MODE) -def test_entire_notebook() -> None: - pass +def test_entire_notebook_trailing_newline() -> None: + with open( + os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_ipynb_string(content, mode=DEFAULT_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {\n' + ' "interpreter": {\n' + ' "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8"\n' # noqa:B950 + " },\n" + ' "kernelspec": {\n' + ' "display_name": "Python 3.8.10 64-bit (\'black\': venv)",\n' + ' "name": "python3"\n' + " },\n" + ' "language_info": {\n' + ' "name": "python",\n' + ' "version": ""\n' + " }\n" + " },\n" + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}\n" + ) + assert result == expected + + +def test_entire_notebook_no_trailing_newline() -> None: + with open( + os.path.join("tests", "data", "notebook_no_trailing_newline.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_ipynb_string(content, mode=DEFAULT_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {\n' + ' "interpreter": {\n' + ' "hash": "e758f3098b5b55f4d87fe30bbdc1367f20f246b483f96267ee70e6c40cb185d8"\n' # noqa: B950 + " },\n" + ' "kernelspec": {\n' + ' "display_name": "Python 3.8.10 64-bit (\'black\': venv)",\n' + ' "name": "python3"\n' + " },\n" + ' "language_info": {\n' + ' "name": "python",\n' + ' "version": ""\n' + " }\n" + " },\n" + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}" + ) + assert result == expected + + +def test_entire_notebook_without_changes() -> None: + with open( + os.path.join("tests", "data", "notebook_without_changes.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + with pytest.raises(NothingChanged): + format_ipynb_string(content, mode=DEFAULT_MODE) + + +def test_empty_string() -> None: + with pytest.raises(NothingChanged): + format_ipynb_string("", mode=DEFAULT_MODE) From 6fa73bc0c0edba44cdbc2f637c55ec3a733bbe6a Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 13:22:32 +0100 Subject: [PATCH 08/81] docstrings --- src/black/__init__.py | 8 +- src/black/handle_ipynb_magics.py | 126 ++++++++++++++++++++++--------- 2 files changed, 95 insertions(+), 39 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0db4a72e8e7..8d7372b37d4 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -847,7 +847,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo `mode` is passed to :func:`format_str`. """ if mode.is_ipynb: - return format_ipynb_string(src_contents, mode=mode, fast=fast) + return format_ipynb_string(src_contents, mode=mode) if not src_contents.strip(): raise NothingChanged @@ -873,6 +873,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo def format_cell(src: str, *, mode: Mode) -> str: + """Format code is given cell of Jupyter notebook.""" src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) @@ -890,9 +891,8 @@ def format_cell(src: str, *, mode: Mode) -> str: return new_src -def format_ipynb_string( - src_contents: str, *, mode: Mode, fast: bool = False -) -> FileContent: +def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: + """Format Jupyter notebook.""" if not src_contents: raise NothingChanged nb = json.loads(src_contents) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index c7b678b5c8d..b2a0466f362 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -18,6 +18,7 @@ class UnsupportedMagic(UserWarning): def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: + """Removing trailing semicolon from Jupyter notebook cell.""" from tokenize_rt import ( src_to_tokens, tokens_to_src, @@ -40,6 +41,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: + """Put trailing semicolon back if cell originally had it.""" from tokenize_rt import ( src_to_tokens, tokens_to_src, @@ -61,14 +63,28 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: def mask_cell(src: str) -> Tuple[str, List[Replacement]]: + """Mask IPython magics so content becomes parseable Python code. + + For example, + + %matplotlib inline + 'foo' + + becomes + + str("dfa9bb") + 'foo' + + The replacements are returned, along with the transformed code. + """ replacements: List[Replacement] = [] try: ast.parse(src) except SyntaxError: - # Might be able to parse it with IPython + # Might have IPython magics, will process below. pass else: - # Syntax is fine, nothing to mask + # Syntax is fine, nothing to mask, early return. return src, replacements from IPython.core.inputtransformer2 import TransformerManager @@ -87,11 +103,11 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: raise SyntaxError replacements += magic_replacements - return transformed, replacements def get_token(src: str, *, is_cell_magic: bool = False) -> str: + """Return randomly generated token to mask IPython magic with.""" token = secrets.token_hex(3) while token in src: # pragma: nocover token = secrets.token_hex(3) @@ -101,6 +117,22 @@ def get_token(src: str, *, is_cell_magic: bool = False) -> str: def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: + """Replace cell magic with token. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, + + get_ipython().run_cell_magic('time', '', 'foo =bar\\n') + + becomes + + # f3b6e1 + foo =bar + + The replacement, along with the transformed code, is returned. + """ replacements: List[Replacement] = [] tree = ast.parse(src) @@ -115,6 +147,21 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: def replace_magics(src: str) -> Tuple[str, List[Replacement]]: + """Replace magics within body of cell. + + Note that 'src' will already have been processed by IPython's + TransformerManager().transform_cell. + + Example, this + + ls =get_ipython().getoutput('ls') + + becomes + + ls =str("a64c67") + + The replacement, along with the transformed code, are returned. + """ replacements = [] tree = ast.parse(src) @@ -137,13 +184,30 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: def unmask_cell(src: str, replacements: List[Replacement]) -> str: + """Remove replacements from cell. + + For example + + # 4fe98a + foo = bar + + becomes + + %%time + foo = bar + """ for replacement in replacements: src = src.replace(replacement.mask, replacement.src) return src def _is_ipython_magic(node: ast.expr) -> bool: - """Check if attribute is IPython magic.""" + """Check if attribute is IPython magic. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + """ return ( isinstance(node, ast.Attribute) and isinstance(node.value, ast.Call) @@ -153,21 +217,19 @@ def _is_ipython_magic(node: ast.expr) -> bool: class CellMagicFinder(ast.NodeVisitor): - """Find cell magics.""" + """Find cell magics. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + """ def __init__(self) -> None: - """Record where cell magics occur.""" self.header: Optional[str] = None self.body: Optional[str] = None def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103 - """ - Find cell magic, extract header and body. - Raises - ------ - AssertionError - Defensive check. - """ + """Find cell magic, extract header and body.""" if ( isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func) @@ -188,23 +250,21 @@ def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103 class MagicFinder(ast.NodeVisitor): - """Visit cell to look for get_ipython calls.""" + """Visit cell to look for get_ipython calls. + + Note that the source of the abstract syntax tree + will already have been processed by IPython's + TransformerManager().transform_cell. + """ def __init__(self) -> None: - """Magics will record where magics occur.""" + """Record where magics occur.""" self.magics: Dict[int, List[Tuple[int, str]]] = collections.defaultdict(list) def visit_Assign(self, node: ast.Assign) -> None: # pylint: disable=C0103,R0912 - """ - Get source to replace ipython magic with. - Parameters - ---------- - node - Function call. - Raises - ------ - AssertionError - Defensive check. + """Look for system assign magics. Example: + + foo = get_ipython().getoutput('ls') """ if ( isinstance(node.value, ast.Call) @@ -227,16 +287,12 @@ def visit_Assign(self, node: ast.Assign) -> None: # pylint: disable=C0103,R0912 self.generic_visit(node) def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103,R0912 - """ - Get source to replace ipython magic with. - Parameters - ---------- - node - Function call. - Raises - ------ - AssertionError - Defensive check. + """Look for magics in body of cell. Examples: + + get_ipython().system('ls') + get_ipython().getoutput('ls') + get_ipython().run_line_magic('pinfo', 'ls') + get_ipython().run_line_magic('pinfo2', 'ls') """ if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): assert isinstance(node.value.func, ast.Attribute) # help mypy From 9e611f54272d16e2ef99d1c20c43e7ec19555287 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 13:50:02 +0100 Subject: [PATCH 09/81] skip multiline magics --- src/black/handle_ipynb_magics.py | 3 +++ tests/test_ipynb.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index b2a0466f362..218e846e493 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -101,6 +101,9 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: except UnsupportedMagic: # will be ignored upstream raise SyntaxError + if len(transformed.splitlines()) != len(src.splitlines()): + # multiline magic, won't format + raise SyntaxError replacements += magic_replacements return transformed, replacements diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 9fd4108e559..0ebeced1315 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -90,6 +90,19 @@ def test_automagic() -> None: format_cell(src, mode=DEFAULT_MODE) +def test_multiline_magic() -> None: + src = "%time 1 + \\\n2" + with pytest.raises(NothingChanged): + format_cell(src, mode=DEFAULT_MODE) + + +def test_multiline_no_magic() -> None: + src = "1 + \\\n2" + result = format_cell(src, mode=DEFAULT_MODE) + expected = "1 + 2" + assert result == expected + + def test_cell_magic_with_invalid_body() -> None: src = "%%time\nif True" with pytest.raises(NothingChanged): From 1f9eccadaf5d6c9150d7dbb8544783fcd931f2ca Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 14:24:21 +0100 Subject: [PATCH 10/81] add test for nested cell magic --- tests/test_ipynb.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 0ebeced1315..1c00763e8f0 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -78,6 +78,13 @@ def test_cell_magic_with_magic() -> None: assert result == expected +def test_cell_magic_nested() -> None: + src = "%%time\n%%time\n2+2" + result = format_cell(src, mode=DEFAULT_MODE) + expected = "%%time\n%%time\n2 + 2" + assert result == expected + + def test_cell_magic_with_magic_noop() -> None: src = "%%t -n1\nls = !ls" with pytest.raises(NothingChanged): From 8bed188ecc5c62a3b7d92d41391e9d16bc79d57f Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 18:59:37 +0100 Subject: [PATCH 11/81] remove ipynb_test.yml, put ipynb tests in tox.ini --- .github/workflows/ipynb_test.yml | 46 -------------------------------- tox.ini | 4 +++ 2 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/ipynb_test.yml diff --git a/.github/workflows/ipynb_test.yml b/.github/workflows/ipynb_test.yml deleted file mode 100644 index 0be99eb184e..00000000000 --- a/.github/workflows/ipynb_test.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: test ipynb - -on: - push: - paths-ignore: - - "docs/**" - - "*.md" - - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. Without this if check, checks are duplicated since - # internal PRs match both the push and pull_request events. - if: - github.event_name == 'push' || github.event.pull_request.head.repo.full_name != - github.repository - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - - - name: Install latest pip - run: | - python -m pip install --upgrade pip - - - name: Test Jupyter Extra Install - run: | - python -m pip install -e ".[jupyter]" - python -m pip install -r test_requirements.txt - - - name: Run Jupyter tests - run: | - pytest tests/test_ipynb.py --cov=black.handle_ipynb_magics --cov-report=term-missing --cov-branch diff --git a/tox.ini b/tox.ini index 3ea4da8eac2..9e7e9ab5510 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,10 @@ commands = pytest tests --run-optional python2 \ !ci: --numprocesses auto \ --cov --cov-append {posargs} + pip install -e .[jupyter] + pytest tests/test_ipynb.py \ + !ci: --numprocesses auto \ + --cov --cov-append {posargs} coverage report [testenv:fuzz] From 8000b03b1a55d5ea8493a00227c9d6ba025691f7 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 19:10:07 +0100 Subject: [PATCH 12/81] add changelog entry --- CHANGES.md | 1 + tests/test_ipynb.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ea5196e07a0..fe879de8db5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - Add primer support and test for code piped into black via STDIN (#2315) - Fix internal error when `FORCE_OPTIONAL_PARENTHESES` feature is enabled (#2332) - Accept empty stdin (#2346) +- Add support for Jupyter Notebook (#2357) ## 21.6b0 diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 1c00763e8f0..5b9b4171cbe 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -3,7 +3,7 @@ from tests.util import DEFAULT_MODE import pytest -pytest.importorskip("IPython") +pytest.importorskip("IPython", reason="IPython is an optional dependency") def test_noop() -> None: From fb03aaaeb21f7f184c51d11a07d4ad9b3f78f4ca Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 19:13:25 +0100 Subject: [PATCH 13/81] typo --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 8d7372b37d4..f262ac6af72 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -873,7 +873,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo def format_cell(src: str, *, mode: Mode) -> str: - """Format code is given cell of Jupyter notebook.""" + """Format code in given cell of Jupyter notebook.""" src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) From 991f85f16b41005d2243a9265dc72bb4c5fb79a7 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 4 Jul 2021 20:04:05 +0100 Subject: [PATCH 14/81] make token same length as magic it replaces --- src/black/handle_ipynb_magics.py | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 218e846e493..8f9eb2f914c 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -72,7 +72,7 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: becomes - str("dfa9bb") + "25716f358c32750e" 'foo' The replacements are returned, along with the transformed code. @@ -109,14 +109,16 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: return transformed, replacements -def get_token(src: str, *, is_cell_magic: bool = False) -> str: +def get_token(src: str, magic: str) -> str: """Return randomly generated token to mask IPython magic with.""" - token = secrets.token_hex(3) + assert magic + nbytes = max(len(magic) // 2 - 1, 1) + token = secrets.token_hex(nbytes) while token in src: # pragma: nocover - token = secrets.token_hex(3) - if is_cell_magic: - return f"# {token}" - return f'str("{token}")' + token = secrets.token_hex(nbytes) + if len(token) + 2 < len(magic): + token = f"{token}." + return f'"{token}"' def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: @@ -127,12 +129,12 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: Example, - get_ipython().run_cell_magic('time', '', 'foo =bar\\n') + get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\n') becomes - # f3b6e1 - foo =bar + "a794." + ls =!ls The replacement, along with the transformed code, is returned. """ @@ -144,7 +146,7 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: cell_magic_finder.visit(tree) if not cell_magic_finder.header: return src, replacements - mask = get_token(src, is_cell_magic=True) + mask = get_token(src, cell_magic_finder.header) replacements.append(Replacement(mask=mask, src=cell_magic_finder.header)) return f"{mask}\n{cell_magic_finder.body}", replacements @@ -157,11 +159,13 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: Example, this - ls =get_ipython().getoutput('ls') + get_ipython().run_line_magic('matplotlib', 'inline') + 'foo' becomes - ls =str("a64c67") + "5e67db56d490fd39" + 'foo' The replacement, along with the transformed code, are returned. """ @@ -179,7 +183,7 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: # defensive check raise UnsupportedMagic col_offset, magic = magic_finder.magics[i][0] - mask = get_token(src) + mask = get_token(src, magic) replacements.append(Replacement(mask=mask, src=magic)) line = line[:col_offset] + mask new_srcs.append(line) @@ -191,7 +195,7 @@ def unmask_cell(src: str, replacements: List[Replacement]) -> str: For example - # 4fe98a + "9b20" foo = bar becomes From 978f6ae8977197cdf57a3e80be62698bc25603a2 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 09:40:23 +0100 Subject: [PATCH 15/81] only include .ipynb by default if jupyter dependencies are found --- src/black/const.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/black/const.py b/src/black/const.py index dbb4826be0e..6a723f7a050 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,10 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" +try: + import IPython # noqa: F401 + import tokenize_rt # noqa: F401 +except ModuleNotFoundError: + DEFAULT_INCLUDES = r"\.pyi?$" +else: + DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" From 9f9c442b020811b62f0f35ad22229440cb5fd944 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 17:30:00 +0100 Subject: [PATCH 16/81] remove logic from const --- src/black/__init__.py | 17 +++++++++++++++-- src/black/const.py | 9 ++------- src/black/handle_ipynb_magics.py | 8 +++++++- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index f262ac6af72..d70a3fd1880 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -30,7 +30,12 @@ from dataclasses import replace import click -from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES +from black.const import ( + DEFAULT_LINE_LENGTH, + DEFAULT_INCLUDES_NO_IPYNB, + DEFAULT_INCLUDES_IPYNB, + DEFAULT_EXCLUDES, +) from black.const import STDIN_PLACEHOLDER from black.nodes import STARS, syms, is_simple_decorator_expression from black.lines import Line, EmptyLineTracker @@ -267,7 +272,6 @@ def validate_regex( @click.option( "--include", type=str, - default=DEFAULT_INCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" @@ -388,6 +392,15 @@ def main( if config and verbose: out(f"Using configuration from {config}.", bold=False, fg="blue") + if include is None: + try: + import IPython # noqa: F401 + import tokenize_rt # noqa: F401 + except ModuleNotFoundError: + include = DEFAULT_INCLUDES_NO_IPYNB + else: + include = DEFAULT_INCLUDES_IPYNB + error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: err( diff --git a/src/black/const.py b/src/black/const.py index 6a723f7a050..8223929a51c 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,10 +1,5 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -try: - import IPython # noqa: F401 - import tokenize_rt # noqa: F401 -except ModuleNotFoundError: - DEFAULT_INCLUDES = r"\.pyi?$" -else: - DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" +DEFAULT_INCLUDES_NO_IPYNB = r"\.pyi?$" +DEFAULT_INCLUDES_IPYNB = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 8f9eb2f914c..46b20ae2965 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -110,7 +110,13 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: def get_token(src: str, magic: str) -> str: - """Return randomly generated token to mask IPython magic with.""" + """Return randomly generated token to mask IPython magic with. + + For example, if 'magic' was `%matplotlib inline`, then a possible + token to mask it with would be `"43fdd17f7e5ddc83"`. The token + will be the same length as the magic, and it may not already be + present in the rest of the cell. + """ assert magic nbytes = max(len(magic) // 2 - 1, 1) token = secrets.token_hex(nbytes) From 85d34cdd6d07c5b7b59c4872370ad669b5b8f293 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 17:32:33 +0100 Subject: [PATCH 17/81] fixup --- tests/test_black.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 42ac119324c..a5d7f771545 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1800,7 +1800,7 @@ def test_extend_exclude(self) -> None: black.gen_python_files( path.iterdir(), this_abs, - re.compile(black.DEFAULT_INCLUDES), + re.compile(black.DEFAULT_INCLUDES_NO_IPYNB), re.compile(r"\.pyi$"), re.compile(r"\.definitely_exclude"), None, @@ -1857,7 +1857,7 @@ def test_symlink_out_of_root_directory(self) -> None: path = MagicMock() root = THIS_DIR.resolve() child = MagicMock() - include = re.compile(black.DEFAULT_INCLUDES) + include = re.compile(black.DEFAULT_INCLUDES_NO_IPYNB) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) From fda607b010951abd0fdc158d36e781ac73e3bca5 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 17:35:19 +0100 Subject: [PATCH 18/81] fixup --- src/black/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index d70a3fd1880..8f473510e1c 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -397,9 +397,9 @@ def main( import IPython # noqa: F401 import tokenize_rt # noqa: F401 except ModuleNotFoundError: - include = DEFAULT_INCLUDES_NO_IPYNB + include = validate_regex(DEFAULT_INCLUDES_NO_IPYNB) else: - include = DEFAULT_INCLUDES_IPYNB + include = validate_regex(DEFAULT_INCLUDES_IPYNB) error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: From 7d77e7d1f716c5aa1184c5fdac82a90d53fee8c0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 17:39:24 +0100 Subject: [PATCH 19/81] re.compile --- src/black/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 8f473510e1c..92c958a7fe2 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -397,9 +397,9 @@ def main( import IPython # noqa: F401 import tokenize_rt # noqa: F401 except ModuleNotFoundError: - include = validate_regex(DEFAULT_INCLUDES_NO_IPYNB) + include = re.compile(DEFAULT_INCLUDES_NO_IPYNB) else: - include = validate_regex(DEFAULT_INCLUDES_IPYNB) + include = re.compile(DEFAULT_INCLUDES_IPYNB) error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: From 3eb55fff0a82f0113c7598f94a249e748ec1bca4 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 17:51:38 +0100 Subject: [PATCH 20/81] noop From 603821c0a6db09a9b8aa8b5dda21ef23d87666db Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 5 Jul 2021 22:50:10 +0100 Subject: [PATCH 21/81] clear up --- src/black/__init__.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 92c958a7fe2..19029cf19aa 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -886,7 +886,17 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo def format_cell(src: str, *, mode: Mode) -> str: - """Format code in given cell of Jupyter notebook.""" + """Format code in given cell of Jupyter notebook. + + General idea is: + + - if cell has trailing semicolon, remove it; + - if cell has IPython magics, mask them; + - format cell; + - reinstate IPython magics; + - reinstate trailing semicolon (if originally present); + - strip trailing newlines. + """ src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) @@ -908,28 +918,28 @@ def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: """Format Jupyter notebook.""" if not src_contents: raise NothingChanged - nb = json.loads(src_contents) trailing_newline = src_contents[-1] == "\n" modified = False - for _, cell in enumerate(nb["cells"]): + nb = json.loads(src_contents) + for cell in nb["cells"]: if cell.get("cell_type", None) == "code": try: src = "".join(cell["source"]) - new_src = format_cell(src, mode=mode) + dst = format_cell(src, mode=mode) except NothingChanged: pass else: - cell["source"] = new_src.splitlines(keepends=True) + cell["source"] = dst.splitlines(keepends=True) modified = True if modified: - res = json.dumps(nb, indent=1, ensure_ascii=False) + dst_contents = json.dumps(nb, indent=1, ensure_ascii=False) if trailing_newline: - res = res + "\n" - if res == src_contents: # pragma: nocover + dst_contents = dst_contents + "\n" + if dst_contents == src_contents: # pragma: nocover # Defensive check raise NothingChanged - return res + return dst_contents else: raise NothingChanged From eff0df5633a8f0d1586425507fa8be1658ff5c0f Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Tue, 6 Jul 2021 11:15:26 +0100 Subject: [PATCH 22/81] new_src -> dst --- src/black/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 19029cf19aa..5156b493fb7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -907,11 +907,11 @@ def format_cell(src: str, *, mode: Mode) -> str: raise NothingChanged formatted_masked_cell = format_str(masked_cell, mode=mode) formatted_cell = unmask_cell(formatted_masked_cell, replacements) - new_src = put_trailing_semicolon_back(formatted_cell, has_trailing_semicolon) - new_src = new_src.rstrip("\n") - if new_src == src: + dst = put_trailing_semicolon_back(formatted_cell, has_trailing_semicolon) + dst = dst.rstrip("\n") + if dst == src: raise NothingChanged - return new_src + return dst def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: From 742a667192e96be7dfd3600396b684718c070f91 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Tue, 6 Jul 2021 11:54:41 +0100 Subject: [PATCH 23/81] early exit for non-python notebooks --- src/black/__init__.py | 2 ++ tests/test_ipynb.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/black/__init__.py b/src/black/__init__.py index 5156b493fb7..993794fe85d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -921,6 +921,8 @@ def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: trailing_newline = src_contents[-1] == "\n" modified = False nb = json.loads(src_contents) + if nb.get("metadata", {}).get("language_info", {}).get("name", None) != "python": + raise NothingChanged for cell in nb["cells"]: if cell.get("cell_type", None) == "code": try: diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 5b9b4171cbe..4f505b01be5 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -234,6 +234,14 @@ def test_entire_notebook_without_changes() -> None: format_ipynb_string(content, mode=DEFAULT_MODE) +def test_non_python_notebook() -> None: + with open(os.path.join("tests", "data", "non_python_notebook.ipynb"), "rb") as fd: + content_bytes = fd.read() + content = content_bytes.decode() + with pytest.raises(NothingChanged): + format_ipynb_string(content, mode=DEFAULT_MODE) + + def test_empty_string() -> None: with pytest.raises(NothingChanged): format_ipynb_string("", mode=DEFAULT_MODE) From 57e15771f11fab7857ad821513d3a7679ce2636d Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Tue, 6 Jul 2021 11:56:04 +0100 Subject: [PATCH 24/81] add non-python test notebook --- tests/data/non_python_notebook.ipynb | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/data/non_python_notebook.ipynb diff --git a/tests/data/non_python_notebook.ipynb b/tests/data/non_python_notebook.ipynb new file mode 100644 index 00000000000..da5cdd8e185 --- /dev/null +++ b/tests/data/non_python_notebook.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(tidyverse) ","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle"},"execution_count":null,"outputs":[]}]} \ No newline at end of file From 58ea513c928966835af32fb984dc5bbaebbe31d4 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 7 Jul 2021 10:59:20 +0100 Subject: [PATCH 25/81] add repo with many notebooks to black-primer --- src/black_primer/primer.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 90643987942..af2fc4e3500 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -76,6 +76,13 @@ "long_checkout": false, "py_versions": ["all"] }, + "planetary computer examples": { + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": false, + "git_clone_url": "git@github.com:microsoft/PlanetaryComputerExamples.git", + "long_checkout": false, + "py_versions": ["all"] + }, "poetry": { "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, From 2ab8ca2144b7476615ecee17cfdde0cf0f81410b Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 7 Jul 2021 11:08:04 +0100 Subject: [PATCH 26/81] install extra dependencies for black-primer --- .github/workflows/primer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/primer.yml b/.github/workflows/primer.yml index 5f41c301737..cd0b4bde6af 100644 --- a/.github/workflows/primer.yml +++ b/.github/workflows/primer.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e ".[d]" + python -m pip install -e ".[d,jupyter]" - name: Primer run env: From d98e49fa3c051bd863d30d0a0759cf805dfeaa72 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 7 Jul 2021 11:11:56 +0100 Subject: [PATCH 27/81] fix planetary computer examples url --- src/black_primer/primer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index af2fc4e3500..52a4ea56d85 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -79,7 +79,7 @@ "planetary computer examples": { "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, - "git_clone_url": "git@github.com:microsoft/PlanetaryComputerExamples.git", + "git_clone_url": "https://github.com/microsoft/PlanetaryComputerExamples", "long_checkout": false, "py_versions": ["all"] }, From 965ef5003164de8ba945e9955920a46c8afd5359 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 21:55:03 +0100 Subject: [PATCH 28/81] dont run on ipynb files by default --- src/black/__init__.py | 13 ++----------- src/black/const.py | 3 +-- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 993794fe85d..0f96c57fae9 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -32,8 +32,7 @@ from black.const import ( DEFAULT_LINE_LENGTH, - DEFAULT_INCLUDES_NO_IPYNB, - DEFAULT_INCLUDES_IPYNB, + DEFAULT_INCLUDES, DEFAULT_EXCLUDES, ) from black.const import STDIN_PLACEHOLDER @@ -272,6 +271,7 @@ def validate_regex( @click.option( "--include", type=str, + default=DEFAULT_INCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" @@ -392,15 +392,6 @@ def main( if config and verbose: out(f"Using configuration from {config}.", bold=False, fg="blue") - if include is None: - try: - import IPython # noqa: F401 - import tokenize_rt # noqa: F401 - except ModuleNotFoundError: - include = re.compile(DEFAULT_INCLUDES_NO_IPYNB) - else: - include = re.compile(DEFAULT_INCLUDES_IPYNB) - error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: err( diff --git a/src/black/const.py b/src/black/const.py index 8223929a51c..821258588ab 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,5 +1,4 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES_NO_IPYNB = r"\.pyi?$" -DEFAULT_INCLUDES_IPYNB = r"(\.pyi?|\.ipynb)$" +DEFAULT_INCLUDES = r"\.pyi?$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" From 083e7948d752707d9c70f142be8fa6aaaad020d9 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 22:00:09 +0100 Subject: [PATCH 29/81] add scikit-lego (Expected to change) to black-primer --- src/black_primer/primer.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 52a4ea56d85..78676d02c2c 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -118,6 +118,13 @@ "long_checkout": false, "py_versions": ["all"] }, + "scikit-lego": { + "cli_arguments": ["--experimental-string-processing", "--include", "\\.ipynb$"], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/koaning/scikit-lego", + "long_checkout": false, + "py_versions": ["all"] + }, "sqlalchemy": { "no_cli_args_reason": "breaks black with new string parsing - #2188", "cli_arguments": [], From d3febc15c8de2a73c900588661df8fb4a149f969 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 22:38:54 +0100 Subject: [PATCH 30/81] add ipynb-specific diff --- src/black/__init__.py | 7 +++++-- src/black/output.py | 18 ++++++++++++++++++ tests/test_ipynb.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0f96c57fae9..97f382ac766 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -44,7 +44,7 @@ from black.mode import Feature, supports_feature, VERSION_TO_FEATURES from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache from black.concurrency import cancel, shutdown, maybe_install_uvloop -from black.output import dump_to_file, diff, color_diff, out, err +from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err from black.report import Report, Changed from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore @@ -773,7 +773,10 @@ def format_file_in_place( now = datetime.utcnow() src_name = f"{src}\t{then} +0000" dst_name = f"{src}\t{now} +0000" - diff_contents = diff(src_contents, dst_contents, src_name, dst_name) + if src.suffix == ".ipynb": + diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) + else: + diff_contents = diff(src_contents, dst_contents, src_name, dst_name) if write_back == WriteBack.COLOR_DIFF: diff_contents = color_diff(diff_contents) diff --git a/src/black/output.py b/src/black/output.py index c253c85e90e..175b83a30bc 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -3,6 +3,7 @@ The double calls are for patching purposes in tests. """ +import json from typing import Any, Optional from mypy_extensions import mypyc_attr import tempfile @@ -34,6 +35,23 @@ def err(message: Optional[str] = None, nl: bool = True, **styles: Any) -> None: _err(message, nl=nl, **styles) +def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: + """Return a unified diff string between each cell in notebooks `a` and `b`.""" + a_nb = json.loads(a) + b_nb = json.loads(b) + diff_lines = [] + cells = (cell for cell in a_nb["cells"] if cell["cell_type"] == "code") + for cell_number, _ in enumerate(cells): + cell_diff = diff( + "".join(a_nb["cells"][cell_number]["source"]) + "\n", + "".join(b_nb["cells"][cell_number]["source"]) + "\n", + f"{a_name}:cell_{cell_number}", + f"{b_name}:cell_{cell_number}", + ) + diff_lines.append(cell_diff) + return "".join(diff_lines) + + def diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between strings `a` and `b`.""" import difflib diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 4f505b01be5..29d2daa2135 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -2,6 +2,7 @@ import os from tests.util import DEFAULT_MODE import pytest +import subprocess pytest.importorskip("IPython", reason="IPython is an optional dependency") @@ -245,3 +246,34 @@ def test_non_python_notebook() -> None: def test_empty_string() -> None: with pytest.raises(NothingChanged): format_ipynb_string("", mode=DEFAULT_MODE) + + +def test_ipynb_diff_with_change() -> None: + output = subprocess.run( + [ + "black", + os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), + "--diff", + ], + stdout=subprocess.PIPE, + universal_newlines=True, + ) + # Ignore the first two lines of output as they contain the current UTC time + result = "".join(output.stdout.splitlines(keepends=True)[2:]) + expected = "@@ -1,3 +1,3 @@\n" " %%time\n" " \n" "-print('foo')\n" '+print("foo")\n' + assert result == expected + + +def test_ipynb_diff_with_no_change() -> None: + output = subprocess.run( + [ + "black", + os.path.join("tests", "data", "notebook_without_changes.ipynb"), + "--diff", + ], + stdout=subprocess.PIPE, + universal_newlines=True, + ) + result = output.stdout + expected = "" + assert result == expected From 62cce53f02a280bc94445e8e40fdeec65e75addd Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 22:43:20 +0100 Subject: [PATCH 31/81] fixup --- tests/test_black.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index a5d7f771545..42ac119324c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1800,7 +1800,7 @@ def test_extend_exclude(self) -> None: black.gen_python_files( path.iterdir(), this_abs, - re.compile(black.DEFAULT_INCLUDES_NO_IPYNB), + re.compile(black.DEFAULT_INCLUDES), re.compile(r"\.pyi$"), re.compile(r"\.definitely_exclude"), None, @@ -1857,7 +1857,7 @@ def test_symlink_out_of_root_directory(self) -> None: path = MagicMock() root = THIS_DIR.resolve() child = MagicMock() - include = re.compile(black.DEFAULT_INCLUDES_NO_IPYNB) + include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) From f786a38f3c0c95f525b3843376025af1fc198b82 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 23:14:06 +0100 Subject: [PATCH 32/81] run on all (including ipynb) by default --- src/black/__init__.py | 13 +++++++++++-- src/black/const.py | 3 ++- tests/test_black.py | 4 ++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 97f382ac766..254acf46167 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -32,7 +32,8 @@ from black.const import ( DEFAULT_LINE_LENGTH, - DEFAULT_INCLUDES, + DEFAULT_INCLUDES_NO_IPYNB, + DEFAULT_INCLUDES_IPYNB, DEFAULT_EXCLUDES, ) from black.const import STDIN_PLACEHOLDER @@ -271,7 +272,6 @@ def validate_regex( @click.option( "--include", type=str, - default=DEFAULT_INCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" @@ -392,6 +392,15 @@ def main( if config and verbose: out(f"Using configuration from {config}.", bold=False, fg="blue") + if include is None: + try: + import IPython # noqa: F401 + import tokenize_rt # noqa: F401 + except ModuleNotFoundError: + include = re.compile(DEFAULT_INCLUDES_NO_IPYNB) + else: + include = re.compile(DEFAULT_INCLUDES_IPYNB) + error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: err( diff --git a/src/black/const.py b/src/black/const.py index 821258588ab..8223929a51c 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,5 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES = r"\.pyi?$" +DEFAULT_INCLUDES_NO_IPYNB = r"\.pyi?$" +DEFAULT_INCLUDES_IPYNB = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/tests/test_black.py b/tests/test_black.py index 42ac119324c..a5d7f771545 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1800,7 +1800,7 @@ def test_extend_exclude(self) -> None: black.gen_python_files( path.iterdir(), this_abs, - re.compile(black.DEFAULT_INCLUDES), + re.compile(black.DEFAULT_INCLUDES_NO_IPYNB), re.compile(r"\.pyi$"), re.compile(r"\.definitely_exclude"), None, @@ -1857,7 +1857,7 @@ def test_symlink_out_of_root_directory(self) -> None: path = MagicMock() root = THIS_DIR.resolve() child = MagicMock() - include = re.compile(black.DEFAULT_INCLUDES) + include = re.compile(black.DEFAULT_INCLUDES_NO_IPYNB) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) From 8853aa3167e7eee8580842c414fff582b34f508f Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 7 Jul 2021 23:56:01 +0100 Subject: [PATCH 33/81] remove --include .ipynb from scikit-lego black-primer --- src/black_primer/primer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 78676d02c2c..c04b1884cd1 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -119,7 +119,7 @@ "py_versions": ["all"] }, "scikit-lego": { - "cli_arguments": ["--experimental-string-processing", "--include", "\\.ipynb$"], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/koaning/scikit-lego", "long_checkout": false, From 879cd113a7a962a9eecc48db5852fd47d71007cc Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 12:14:21 +0100 Subject: [PATCH 34/81] use tokenize so as to mirror the exact logic in IPython.core.displayhooks quiet --- setup.py | 2 +- src/black/__init__.py | 20 ++++++------ src/black/const.py | 3 +- src/black/handle_ipynb_magics.py | 54 +++++++++++++++++--------------- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/setup.py b/setup.py index 4e024fdc2ec..cfb043ed246 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ def get_long_description() -> str: "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], "uvloop": ["uvloop>=0.15.2"], - "jupyter": ["ipython>=7.8.0", "tokenize-rt>=3.2.0"], + "jupyter": ["ipython>=7.8.0"], }, test_suite="tests.test_black", classifiers=[ diff --git a/src/black/__init__.py b/src/black/__init__.py index 254acf46167..2c5f799e3bf 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,4 +1,5 @@ import asyncio +import warnings import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor from contextlib import contextmanager @@ -32,8 +33,7 @@ from black.const import ( DEFAULT_LINE_LENGTH, - DEFAULT_INCLUDES_NO_IPYNB, - DEFAULT_INCLUDES_IPYNB, + DEFAULT_INCLUDES, DEFAULT_EXCLUDES, ) from black.const import STDIN_PLACEHOLDER @@ -272,6 +272,7 @@ def validate_regex( @click.option( "--include", type=str, + default=DEFAULT_INCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" @@ -392,15 +393,6 @@ def main( if config and verbose: out(f"Using configuration from {config}.", bold=False, fg="blue") - if include is None: - try: - import IPython # noqa: F401 - import tokenize_rt # noqa: F401 - except ModuleNotFoundError: - include = re.compile(DEFAULT_INCLUDES_NO_IPYNB) - else: - include = re.compile(DEFAULT_INCLUDES_IPYNB) - error_msg = "Oh no! 💥 💔 💥" if required_version and required_version != __version__: err( @@ -774,6 +766,12 @@ def format_file_in_place( dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) except NothingChanged: return False + except ModuleNotFoundError: + warnings.warn( + f"Skipping '{src}' as extra dependencies are not installed.\n" + "You can fix this with ``pip install black[jupyter]``" + ) + return False if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: diff --git a/src/black/const.py b/src/black/const.py index 8223929a51c..dbb4826be0e 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,5 +1,4 @@ DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 -DEFAULT_INCLUDES_NO_IPYNB = r"\.pyi?$" -DEFAULT_INCLUDES_IPYNB = r"(\.pyi?|\.ipynb)$" +DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 46b20ae2965..16f999942e4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,4 +1,6 @@ import ast +import tokenize +import io from typing import Dict import secrets @@ -18,48 +20,50 @@ class UnsupportedMagic(UserWarning): def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: - """Removing trailing semicolon from Jupyter notebook cell.""" - from tokenize_rt import ( - src_to_tokens, - tokens_to_src, - NON_CODING_TOKENS, - reversed_enumerate, - ) + """Remove trailing semicolon from Jupyter notebook cell. - tokens = src_to_tokens(src) + Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. + """ + tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))[::-1] trailing_semicolon = False - for idx, token in reversed_enumerate(tokens): - if token.name in NON_CODING_TOKENS or token.name == "NEWLINE" or not token.src: + for token in tokens: + if token[0] in ( + tokenize.ENDMARKER, + tokenize.NL, + tokenize.NEWLINE, + tokenize.COMMENT, + ): continue - if token.name == "OP" and token.src == ";": - del tokens[idx] + if token[0] == tokenize.OP and token[1] == ";": + del token trailing_semicolon = True break if not trailing_semicolon: return src, False - return tokens_to_src(tokens), True + return tokenize.untokenize(tokens[::-1]), True def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: - """Put trailing semicolon back if cell originally had it.""" - from tokenize_rt import ( - src_to_tokens, - tokens_to_src, - NON_CODING_TOKENS, - reversed_enumerate, - ) + """Put trailing semicolon back if cell originally had it. + Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. + """ if not has_trailing_semicolon: return src - tokens = src_to_tokens(src) - for idx, token in reversed_enumerate(tokens): - if token.name in NON_CODING_TOKENS or token.name == "NEWLINE" or not token.src: + tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))[::-1] + for idx, token in enumerate(tokens): + if token[0] in ( + tokenize.ENDMARKER, + tokenize.NL, + tokenize.NEWLINE, + tokenize.COMMENT, + ): continue - tokens[idx] = token._replace(src=token.src + ";") + tokens[idx] = token._replace(string=token.string + ";") break else: # pragma: nocover raise AssertionError("Unreachable code") - return str(tokens_to_src(tokens)) + return str(tokenize.untokenize(tokens[::-1])) def mask_cell(src: str) -> Tuple[str, List[Replacement]]: From 671696331ddfa14183daa2869210da6bd0dc03cb Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 12:16:47 +0100 Subject: [PATCH 35/81] fixup --- tests/test_black.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index a5d7f771545..42ac119324c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1800,7 +1800,7 @@ def test_extend_exclude(self) -> None: black.gen_python_files( path.iterdir(), this_abs, - re.compile(black.DEFAULT_INCLUDES_NO_IPYNB), + re.compile(black.DEFAULT_INCLUDES), re.compile(r"\.pyi$"), re.compile(r"\.definitely_exclude"), None, @@ -1857,7 +1857,7 @@ def test_symlink_out_of_root_directory(self) -> None: path = MagicMock() root = THIS_DIR.resolve() child = MagicMock() - include = re.compile(black.DEFAULT_INCLUDES_NO_IPYNB) + include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) From 786ac0611e5adcabbf130f0cb9704d98f7a995ae Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 12:17:57 +0100 Subject: [PATCH 36/81] :art: --- src/black/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 2c5f799e3bf..bd1f8c43456 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -31,11 +31,7 @@ from dataclasses import replace import click -from black.const import ( - DEFAULT_LINE_LENGTH, - DEFAULT_INCLUDES, - DEFAULT_EXCLUDES, -) +from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES from black.const import STDIN_PLACEHOLDER from black.nodes import STARS, syms, is_simple_decorator_expression from black.lines import Line, EmptyLineTracker From 48b56e1ac412e85f33c422d54057e4c53c424687 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 12:20:26 +0100 Subject: [PATCH 37/81] clarify docstring --- src/black/handle_ipynb_magics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 16f999942e4..daf984f2e17 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -118,8 +118,8 @@ def get_token(src: str, magic: str) -> str: For example, if 'magic' was `%matplotlib inline`, then a possible token to mask it with would be `"43fdd17f7e5ddc83"`. The token - will be the same length as the magic, and it may not already be - present in the rest of the cell. + will be the same length as the magic, and we make sure that it was + not already present anywhere else in the cell. """ assert magic nbytes = max(len(magic) // 2 - 1, 1) From c74959dd34f59d089480d5e4d2fb61656a0cdd95 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 14:46:46 +0100 Subject: [PATCH 38/81] add test for when comment is after trailing semicolon --- src/black/handle_ipynb_magics.py | 14 ++++++++++++-- tests/test_ipynb.py | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index daf984f2e17..1fb4b3a7175 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -22,11 +22,21 @@ class UnsupportedMagic(UserWarning): def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: """Remove trailing semicolon from Jupyter notebook cell. + For example, + + fig, ax = plt.subplots() + ax.plot(x_data, y_data); # plot data + + would become + + fig, ax = plt.subplots() + ax.plot(x_data, y_data) # plot data + Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. """ tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))[::-1] trailing_semicolon = False - for token in tokens: + for idx, token in enumerate(tokens): if token[0] in ( tokenize.ENDMARKER, tokenize.NL, @@ -35,7 +45,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: ): continue if token[0] == tokenize.OP and token[1] == ";": - del token + del tokens[idx] trailing_semicolon = True break if not trailing_semicolon: diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 29d2daa2135..b0d87271045 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -20,6 +20,13 @@ def test_trailing_semicolon() -> None: assert result == expected +def test_trailing_semicolon_with_comment() -> None: + src = 'foo = "a" ; # bar' + result = format_cell(src, mode=DEFAULT_MODE) + expected = 'foo = "a"; # bar' + assert result == expected + + def test_trailing_semicolon_noop() -> None: src = 'foo = "a";' with pytest.raises(NothingChanged): From 9e3b9bd16cd4856a0ae6bd2aa664721af68f27c0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 15:18:46 +0100 Subject: [PATCH 39/81] enumerate(reversed) instead of [::-1] --- src/black/handle_ipynb_magics.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 1fb4b3a7175..5fc1fdbf6f4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -34,9 +34,9 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. """ - tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))[::-1] + tokens = list(tokenize.generate_tokens(io.StringIO(src).readline)) trailing_semicolon = False - for idx, token in enumerate(tokens): + for idx, token in enumerate(reversed(tokens), start=1): if token[0] in ( tokenize.ENDMARKER, tokenize.NL, @@ -45,12 +45,13 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: ): continue if token[0] == tokenize.OP and token[1] == ";": - del tokens[idx] + # We're iterating backwards, so `-idx`. + del tokens[-idx] trailing_semicolon = True break if not trailing_semicolon: return src, False - return tokenize.untokenize(tokens[::-1]), True + return tokenize.untokenize(tokens), True def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: @@ -60,8 +61,8 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: """ if not has_trailing_semicolon: return src - tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))[::-1] - for idx, token in enumerate(tokens): + tokens = list(tokenize.generate_tokens(io.StringIO(src).readline)) + for idx, token in enumerate(reversed(tokens), start=1): if token[0] in ( tokenize.ENDMARKER, tokenize.NL, @@ -69,11 +70,12 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: tokenize.COMMENT, ): continue - tokens[idx] = token._replace(string=token.string + ";") + # We're iterating backwards, so `-idx`. + tokens[-idx] = token._replace(string=token.string + ";") break else: # pragma: nocover raise AssertionError("Unreachable code") - return str(tokenize.untokenize(tokens[::-1])) + return str(tokenize.untokenize(tokens)) def mask_cell(src: str) -> Tuple[str, List[Replacement]]: From 200669f471f5004d06fb845427a26074ed57f3d0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Thu, 8 Jul 2021 18:52:48 +0100 Subject: [PATCH 40/81] clarify docstrings --- src/black/__init__.py | 24 ++++++---- src/black/handle_ipynb_magics.py | 81 ++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index bd1f8c43456..894473e175f 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -893,18 +893,19 @@ def format_cell(src: str, *, mode: Mode) -> str: - reinstate IPython magics; - reinstate trailing semicolon (if originally present); - strip trailing newlines. + + Cells with syntax errors will not be processed, as they + could potentially be automagics or multi-line magics, which + are currently not supported. """ - src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( - src - ) + dst, has_trailing_semicolon = remove_trailing_semicolon(src) try: - masked_cell, replacements = mask_cell(src_without_trailing_semicolon) + dst, replacements = mask_cell(dst) except SyntaxError: - # Don't format, might be automagic or multi-line magic. raise NothingChanged - formatted_masked_cell = format_str(masked_cell, mode=mode) - formatted_cell = unmask_cell(formatted_masked_cell, replacements) - dst = put_trailing_semicolon_back(formatted_cell, has_trailing_semicolon) + dst = format_str(dst, mode=mode) + dst = unmask_cell(dst, replacements) + dst = put_trailing_semicolon_back(dst, has_trailing_semicolon) dst = dst.rstrip("\n") if dst == src: raise NothingChanged @@ -912,7 +913,11 @@ def format_cell(src: str, *, mode: Mode) -> str: def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: - """Format Jupyter notebook.""" + """Format Jupyter notebook. + + Operate cell-by-cell, only on code cells, only for Python notebooks. + If the ``.ipynb`` originally had a trailing newline, it'll be preseved. + """ if not src_contents: raise NothingChanged trailing_newline = src_contents[-1] == "\n" @@ -930,7 +935,6 @@ def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: else: cell["source"] = dst.splitlines(keepends=True) modified = True - if modified: dst_contents = json.dumps(nb, indent=1, ensure_ascii=False) if trailing_newline: diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 5fc1fdbf6f4..91754c6be5d 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,3 +1,4 @@ +"""Functions to process IPython magics with.""" import ast import tokenize import io @@ -16,7 +17,7 @@ class Replacement(NamedTuple): class UnsupportedMagic(UserWarning): - """Raise when Magic (e.g. `a = b??`) is not supported.""" + """Raise when Magic is not supported (e.g. `a = b??`)""" def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: @@ -32,7 +33,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: fig, ax = plt.subplots() ax.plot(x_data, y_data) # plot data - Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. + Mirrors the logic in `quiet` from `IPython.core.displayhook`. """ tokens = list(tokenize.generate_tokens(io.StringIO(src).readline)) trailing_semicolon = False @@ -57,7 +58,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: """Put trailing semicolon back if cell originally had it. - Mirrors the logic in `quiet` from `IPython.core.dispalyhook`. + Mirrors the logic in `quiet` from `IPython.core.displayhook`. """ if not has_trailing_semicolon: return src @@ -107,20 +108,15 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: transformer_manager = TransformerManager() transformed = transformer_manager.transform_cell(src) - transformed, cell_magic_replacements = replace_cell_magics(transformed) replacements += cell_magic_replacements - transformed = transformer_manager.transform_cell(transformed) try: transformed, magic_replacements = replace_magics(transformed) except UnsupportedMagic: - # will be ignored upstream raise SyntaxError - if len(transformed.splitlines()) != len(src.splitlines()): - # multiline magic, won't format + if len(transformed.splitlines()) != len(src.splitlines()): # multi-line magic raise SyntaxError - replacements += magic_replacements return transformed, replacements @@ -192,11 +188,8 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: The replacement, along with the transformed code, are returned. """ replacements = [] - - tree = ast.parse(src) - magic_finder = MagicFinder() - magic_finder.visit(tree) + magic_finder.visit(ast.parse(src)) new_srcs = [] for i, line in enumerate(src.splitlines(), start=1): if i in magic_finder.magics: @@ -204,7 +197,7 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: if len(magics) != 1: # pragma: nocover # defensive check raise UnsupportedMagic - col_offset, magic = magic_finder.magics[i][0] + col_offset, magic = magics[0] mask = get_token(src, magic) replacements.append(Replacement(mask=mask, src=magic)) line = line[:col_offset] + mask @@ -251,13 +244,23 @@ class CellMagicFinder(ast.NodeVisitor): Note that the source of the abstract syntax tree will already have been processed by IPython's TransformerManager().transform_cell. + + For example, + + %%time\nfoo() + + would have been transformed to + + get_ipython().run_cell_magic('time', '', 'foo()\\n') + + and we look for instances of the latter. """ def __init__(self) -> None: self.header: Optional[str] = None self.body: Optional[str] = None - def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103 + def visit_Expr(self, node: ast.Expr) -> None: """Find cell magic, extract header and body.""" if ( isinstance(node.value, ast.Call) @@ -284,16 +287,35 @@ class MagicFinder(ast.NodeVisitor): Note that the source of the abstract syntax tree will already have been processed by IPython's TransformerManager().transform_cell. + + For example, + + %matplotlib inline + + would have been transformed to + + get_ipython().run_line_magic('matplotlib', 'inline') + + and we look for instances of the latter (and likewise for other + types of magics). """ def __init__(self) -> None: """Record where magics occur.""" self.magics: Dict[int, List[Tuple[int, str]]] = collections.defaultdict(list) - def visit_Assign(self, node: ast.Assign) -> None: # pylint: disable=C0103,R0912 - """Look for system assign magics. Example: + def visit_Assign(self, node: ast.Assign) -> None: + """Look for system assign magics. + + For example, - foo = get_ipython().getoutput('ls') + black_version = !black --version + + would have been transformed to + + black_version = get_ipython().getoutput('black --version') + + and we look for instances of the latter. """ if ( isinstance(node.value, ast.Call) @@ -315,13 +337,24 @@ def visit_Assign(self, node: ast.Assign) -> None: # pylint: disable=C0103,R0912 ) self.generic_visit(node) - def visit_Expr(self, node: ast.Expr) -> None: # pylint: disable=C0103,R0912 - """Look for magics in body of cell. Examples: + def visit_Expr(self, node: ast.Expr) -> None: + """Look for magics in body of cell. + + For examples, + + !ls + !!ls + ?ls + ??ls + + would (respectively) get transformed to + + get_ipython().system('ls') + get_ipython().getoutput('ls') + get_ipython().run_line_magic('pinfo', 'ls') + get_ipython().run_line_magic('pinfo2', 'ls') - get_ipython().system('ls') - get_ipython().getoutput('ls') - get_ipython().run_line_magic('pinfo', 'ls') - get_ipython().run_line_magic('pinfo2', 'ls') + and we look for instances of any of the latter. """ if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): assert isinstance(node.value.func, ast.Attribute) # help mypy From db9a8ba15321f20d3b4a701d21a91f9fa5cf95e6 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Fri, 9 Jul 2021 15:24:12 +0100 Subject: [PATCH 41/81] wip --- README.md | 3 ++- pyproject.toml | 1 + src/black/__init__.py | 5 +---- src/black/output.py | 10 +++++----- tests/test_ipynb.py | 1 + 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0f1953df341..307ec9804a0 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,8 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run. If you want to format Python 2 code as well, install with -`pip install black[python2]`. +`pip install black[python2]`. If you want format Jupyter Notebooks as well, install with +`pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/pyproject.toml b/pyproject.toml index 79060fc422d..d085c0ddc62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,4 +31,5 @@ build-backend = "setuptools.build_meta" optional-tests = [ "no_python2: run when `python2` extra NOT installed", "no_blackd: run when `d` extra NOT installed", + "no_jupyter: run when `jupyter` extra NOT installed", ] diff --git a/src/black/__init__.py b/src/black/__init__.py index 894473e175f..786a590d0a0 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -763,10 +763,7 @@ def format_file_in_place( except NothingChanged: return False except ModuleNotFoundError: - warnings.warn( - f"Skipping '{src}' as extra dependencies are not installed.\n" - "You can fix this with ``pip install black[jupyter]``" - ) + warnings.warn(f"Skipping '{src}' as IPython is not installed.\n") return False if write_back == WriteBack.YES: diff --git a/src/black/output.py b/src/black/output.py index 175b83a30bc..fd3dbb37627 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -39,16 +39,16 @@ def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between each cell in notebooks `a` and `b`.""" a_nb = json.loads(a) b_nb = json.loads(b) - diff_lines = [] - cells = (cell for cell in a_nb["cells"] if cell["cell_type"] == "code") - for cell_number, _ in enumerate(cells): - cell_diff = diff( + diff_lines = [ + diff( "".join(a_nb["cells"][cell_number]["source"]) + "\n", "".join(b_nb["cells"][cell_number]["source"]) + "\n", f"{a_name}:cell_{cell_number}", f"{b_name}:cell_{cell_number}", ) - diff_lines.append(cell_diff) + for cell_number, cell in enumerate(a_nb["cells"]) + if cell["cell_type"] == "code" + ] return "".join(diff_lines) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index b0d87271045..ae6c932c497 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -5,6 +5,7 @@ import subprocess pytest.importorskip("IPython", reason="IPython is an optional dependency") +pytestmark = pytest.mark.jupyter def test_noop() -> None: From 18a502ab7493f7aa4348594c71c0f057cfa3d67e Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Fri, 9 Jul 2021 18:19:12 +0100 Subject: [PATCH 42/81] use jupyter and no_jupyter marks --- tests/test_ipynb.py | 1 - tests/test_no_ipynb.py | 15 +++++++++++++++ tox.ini | 5 ++++- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 tests/test_no_ipynb.py diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index ae6c932c497..5d9733c34b7 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -4,7 +4,6 @@ import pytest import subprocess -pytest.importorskip("IPython", reason="IPython is an optional dependency") pytestmark = pytest.mark.jupyter diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py new file mode 100644 index 00000000000..024e362c221 --- /dev/null +++ b/tests/test_no_ipynb.py @@ -0,0 +1,15 @@ +from tests.util import DEFAULT_MODE +import pytest +from black import WriteBack, format_file_in_place +import pathlib + +pytestmark = pytest.mark.no_jupyter + + +def test_ipynb_diff_with_no_change() -> None: + path = pathlib.Path("tests") / "data/notebook_trailing_newline.ipynb" + msg = f"Skipping '{path}' as IPython is not installed." + with pytest.warns(UserWarning, match=msg): + format_file_in_place( + path, fast=False, mode=DEFAULT_MODE, write_back=WriteBack.DIFF + ) diff --git a/tox.ini b/tox.ini index 9e7e9ab5510..2be250d4d9f 100644 --- a/tox.ini +++ b/tox.ini @@ -12,14 +12,17 @@ commands = pip install -e .[d] coverage erase pytest tests --run-optional no_python2 \ + --run-optional no_jupyter \ !ci: --numprocesses auto \ --cov {posargs} pip install -e .[d,python2] pytest tests --run-optional python2 \ + --run-optional no_jupyter \ !ci: --numprocesses auto \ --cov --cov-append {posargs} pip install -e .[jupyter] - pytest tests/test_ipynb.py \ + pytest tests --run-optional jupyter \ + -m jupyter \ !ci: --numprocesses auto \ --cov --cov-append {posargs} coverage report From d6a48692cfa9ed46baaae47d61aef1e73f3cbd02 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Fri, 9 Jul 2021 18:26:07 +0100 Subject: [PATCH 43/81] use THIS_DIR --- tests/test_no_ipynb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index 024e362c221..d1f12339efc 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,13 +1,14 @@ from tests.util import DEFAULT_MODE import pytest from black import WriteBack, format_file_in_place -import pathlib + +from tests.util import THIS_DIR pytestmark = pytest.mark.no_jupyter def test_ipynb_diff_with_no_change() -> None: - path = pathlib.Path("tests") / "data/notebook_trailing_newline.ipynb" + path = THIS_DIR / "data/notebook_trailing_newline.ipynb" msg = f"Skipping '{path}' as IPython is not installed." with pytest.warns(UserWarning, match=msg): format_file_in_place( From e45a2087e1ee7994570c774e424c7053e1de564e Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Fri, 9 Jul 2021 18:30:07 +0100 Subject: [PATCH 44/81] windows fixup --- tests/test_no_ipynb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index d1f12339efc..5ba2b850cb3 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -9,7 +9,7 @@ def test_ipynb_diff_with_no_change() -> None: path = THIS_DIR / "data/notebook_trailing_newline.ipynb" - msg = f"Skipping '{path}' as IPython is not installed." + msg = r"Skipping '.*' as IPython is not installed." with pytest.warns(UserWarning, match=msg): format_file_in_place( path, fast=False, mode=DEFAULT_MODE, write_back=WriteBack.DIFF From 98d1ea3174995071b79c59720b6baac023ff9838 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 11:09:46 +0100 Subject: [PATCH 45/81] perform safe check cell-by-cell for ipynb --- src/black/__init__.py | 67 ++++++++++++++++++++++++------------------- tests/test_ipynb.py | 50 ++++++++++++++++---------------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 786a590d0a0..0234d79a919 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -846,6 +846,23 @@ def format_stdin_to_stdout( f.detach() +def check_src_and_dst_equivalent( + src_contents: str, dst_contents: str, *, mode: Mode +) -> None: + assert_equivalent(src_contents, dst_contents) + + # Forced second pass to work around optional trailing commas (becoming + # forced trailing commas on pass 2) interacting differently with optional + # parentheses. Admittedly ugly. + dst_contents_pass2 = format_str(dst_contents, mode=mode) + if dst_contents != dst_contents_pass2: + dst_contents = dst_contents_pass2 + assert_equivalent(src_contents, dst_contents, pass_num=2) + assert_stable(src_contents, dst_contents, mode=mode) + # Note: no need to explicitly call `assert_stable` if `dst_contents` was + # the same as `dst_contents_pass2`. + + def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Reformat contents of a file and return new contents. @@ -853,33 +870,23 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. `mode` is passed to :func:`format_str`. """ - if mode.is_ipynb: - return format_ipynb_string(src_contents, mode=mode) - if not src_contents.strip(): raise NothingChanged - dst_contents = format_str(src_contents, mode=mode) + if mode.is_ipynb: + dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) + else: + dst_contents = format_str(src_contents, mode=mode) if src_contents == dst_contents: raise NothingChanged - if not fast: - assert_equivalent(src_contents, dst_contents) - - # Forced second pass to work around optional trailing commas (becoming - # forced trailing commas on pass 2) interacting differently with optional - # parentheses. Admittedly ugly. - dst_contents_pass2 = format_str(dst_contents, mode=mode) - if dst_contents != dst_contents_pass2: - dst_contents = dst_contents_pass2 - assert_equivalent(src_contents, dst_contents, pass_num=2) - assert_stable(src_contents, dst_contents, mode=mode) - # Note: no need to explicitly call `assert_stable` if `dst_contents` was - # the same as `dst_contents_pass2`. + if not fast and not mode.is_ipynb: + # ipynb files are checked cell-by-cell + check_src_and_dst_equivalent(src_contents, dst_contents, mode=mode) return dst_contents -def format_cell(src: str, *, mode: Mode) -> str: +def format_cell(src: str, *, fast: bool, mode: Mode) -> str: """Format code in given cell of Jupyter notebook. General idea is: @@ -895,28 +902,31 @@ def format_cell(src: str, *, mode: Mode) -> str: could potentially be automagics or multi-line magics, which are currently not supported. """ - dst, has_trailing_semicolon = remove_trailing_semicolon(src) + src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( + src + ) try: - dst, replacements = mask_cell(dst) + masked_src, replacements = mask_cell(src_without_trailing_semicolon) except SyntaxError: raise NothingChanged - dst = format_str(dst, mode=mode) - dst = unmask_cell(dst, replacements) - dst = put_trailing_semicolon_back(dst, has_trailing_semicolon) + masked_dst = format_str(masked_src, mode=mode) + check_src_and_dst_equivalent(masked_src, masked_dst, mode=mode) + dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements) + dst = put_trailing_semicolon_back( + dst_without_trailing_semicolon, has_trailing_semicolon + ) dst = dst.rstrip("\n") if dst == src: raise NothingChanged return dst -def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: +def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Format Jupyter notebook. Operate cell-by-cell, only on code cells, only for Python notebooks. If the ``.ipynb`` originally had a trailing newline, it'll be preseved. """ - if not src_contents: - raise NothingChanged trailing_newline = src_contents[-1] == "\n" modified = False nb = json.loads(src_contents) @@ -926,7 +936,7 @@ def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: if cell.get("cell_type", None) == "code": try: src = "".join(cell["source"]) - dst = format_cell(src, mode=mode) + dst = format_cell(src, fast=fast, mode=mode) except NothingChanged: pass else: @@ -936,9 +946,6 @@ def format_ipynb_string(src_contents: str, *, mode: Mode) -> FileContent: dst_contents = json.dumps(nb, indent=1, ensure_ascii=False) if trailing_newline: dst_contents = dst_contents + "\n" - if dst_contents == src_contents: # pragma: nocover - # Defensive check - raise NothingChanged return dst_contents else: raise NothingChanged diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 5d9733c34b7..4b9ccaa08e2 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,28 +1,30 @@ -from black import NothingChanged, format_cell, format_ipynb_string +from black import NothingChanged, format_cell, format_file_contents import os -from tests.util import DEFAULT_MODE import pytest import subprocess +from black import Mode pytestmark = pytest.mark.jupyter +JUPYTER_MODE = Mode(is_ipynb=True) + def test_noop() -> None: src = 'foo = "a"' with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_trailing_semicolon() -> None: src = 'foo = "a" ;' - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = 'foo = "a";' assert result == expected def test_trailing_semicolon_with_comment() -> None: src = 'foo = "a" ; # bar' - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = 'foo = "a"; # bar' assert result == expected @@ -30,12 +32,12 @@ def test_trailing_semicolon_with_comment() -> None: def test_trailing_semicolon_noop() -> None: src = 'foo = "a";' with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_cell_magic() -> None: src = "%%time\nfoo =bar" - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = "%%time\nfoo = bar" assert result == expected @@ -43,7 +45,7 @@ def test_cell_magic() -> None: def test_cell_magic_noop() -> None: src = "%%time\n2 + 2" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) @pytest.mark.parametrize( @@ -63,32 +65,32 @@ def test_cell_magic_noop() -> None: ), ) def test_magic(src: str, expected: str) -> None: - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) assert result == expected def test_set_input() -> None: src = "a = b??" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_magic_noop() -> None: src = "ls = !ls" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_cell_magic_with_magic() -> None: src = "%%t -n1\nls =!ls" - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = "%%t -n1\nls = !ls" assert result == expected def test_cell_magic_nested() -> None: src = "%%time\n%%time\n2+2" - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = "%%time\n%%time\n2 + 2" assert result == expected @@ -96,24 +98,24 @@ def test_cell_magic_nested() -> None: def test_cell_magic_with_magic_noop() -> None: src = "%%t -n1\nls = !ls" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_automagic() -> None: src = "pip install black" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_multiline_magic() -> None: src = "%time 1 + \\\n2" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_multiline_no_magic() -> None: src = "1 + \\\n2" - result = format_cell(src, mode=DEFAULT_MODE) + result = format_cell(src, fast=True, mode=JUPYTER_MODE) expected = "1 + 2" assert result == expected @@ -121,13 +123,13 @@ def test_multiline_no_magic() -> None: def test_cell_magic_with_invalid_body() -> None: src = "%%time\nif True" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_empty_cell() -> None: src = "" with pytest.raises(NothingChanged): - format_cell(src, mode=DEFAULT_MODE) + format_cell(src, fast=True, mode=JUPYTER_MODE) def test_entire_notebook_trailing_newline() -> None: @@ -136,7 +138,7 @@ def test_entire_notebook_trailing_newline() -> None: ) as fd: content_bytes = fd.read() content = content_bytes.decode() - result = format_ipynb_string(content, mode=DEFAULT_MODE) + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) expected = ( "{\n" ' "cells": [\n' @@ -187,7 +189,7 @@ def test_entire_notebook_no_trailing_newline() -> None: ) as fd: content_bytes = fd.read() content = content_bytes.decode() - result = format_ipynb_string(content, mode=DEFAULT_MODE) + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) expected = ( "{\n" ' "cells": [\n' @@ -239,7 +241,7 @@ def test_entire_notebook_without_changes() -> None: content_bytes = fd.read() content = content_bytes.decode() with pytest.raises(NothingChanged): - format_ipynb_string(content, mode=DEFAULT_MODE) + format_file_contents(content, fast=True, mode=JUPYTER_MODE) def test_non_python_notebook() -> None: @@ -247,12 +249,12 @@ def test_non_python_notebook() -> None: content_bytes = fd.read() content = content_bytes.decode() with pytest.raises(NothingChanged): - format_ipynb_string(content, mode=DEFAULT_MODE) + format_file_contents(content, fast=True, mode=JUPYTER_MODE) def test_empty_string() -> None: with pytest.raises(NothingChanged): - format_ipynb_string("", mode=DEFAULT_MODE) + format_file_contents("", fast=True, mode=JUPYTER_MODE) def test_ipynb_diff_with_change() -> None: From 93702445a77ee37f399661431483f84383e79ddc Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 11:13:47 +0100 Subject: [PATCH 46/81] only perform safe check in ipynb if not fast --- src/black/__init__.py | 3 ++- tests/test_ipynb.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0234d79a919..3b6ea55b91c 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -910,7 +910,8 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: except SyntaxError: raise NothingChanged masked_dst = format_str(masked_src, mode=mode) - check_src_and_dst_equivalent(masked_src, masked_dst, mode=mode) + if not fast: + check_src_and_dst_equivalent(masked_src, masked_dst, mode=mode) dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements) dst = put_trailing_semicolon_back( dst_without_trailing_semicolon, has_trailing_semicolon diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 4b9ccaa08e2..ec856d7e83a 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -15,9 +15,10 @@ def test_noop() -> None: format_cell(src, fast=True, mode=JUPYTER_MODE) -def test_trailing_semicolon() -> None: +@pytest.mark.parametrize("fast", [True, False]) +def test_trailing_semicolon(fast: bool) -> None: src = 'foo = "a" ;' - result = format_cell(src, fast=True, mode=JUPYTER_MODE) + result = format_cell(src, fast=fast, mode=JUPYTER_MODE) expected = 'foo = "a";' assert result == expected From a6c6341cc2eb80a3c366e22601d80b4bd2a0cd11 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 11:19:00 +0100 Subject: [PATCH 47/81] remove redundant Optional --- src/black/handle_ipynb_magics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 91754c6be5d..e4f1c423c02 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -132,8 +132,12 @@ def get_token(src: str, magic: str) -> str: assert magic nbytes = max(len(magic) // 2 - 1, 1) token = secrets.token_hex(nbytes) + counter = 0 while token in src: # pragma: nocover token = secrets.token_hex(nbytes) + counter += 1 + if counter > 100: + raise AssertionError() if len(token) + 2 < len(magic): token = f"{token}." return f'"{token}"' @@ -272,9 +276,8 @@ def visit_Expr(self, node: ast.Expr) -> None: for arg in node.value.args: assert isinstance(arg, ast.Str) args.append(arg.s) - header: Optional[str] = f"%%{args[0]}" + header = f"%%{args[0]}" if args[1]: - assert header is not None header += f" {args[1]}" self.header = header self.body = args[2] From ddc34e1f5dca7e33c5462528fb2f8d58a3153c8b Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 11:22:33 +0100 Subject: [PATCH 48/81] :art: --- src/black/handle_ipynb_magics.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index e4f1c423c02..f604fde2ed2 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -332,12 +332,7 @@ def visit_Assign(self, node: ast.Assign) -> None: args.append(arg.s) assert args src = f"!{args[0]}" - self.magics[node.value.lineno].append( - ( - node.value.col_offset, - src, - ) - ) + self.magics[node.value.lineno].append((node.value.col_offset, src)) self.generic_visit(node) def visit_Expr(self, node: ast.Expr) -> None: From 75d493a1ecc3e226b18fcf96481c050b6a1b98c6 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 11:29:30 +0100 Subject: [PATCH 49/81] use typeguard --- src/black/handle_ipynb_magics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index f604fde2ed2..6887fb631e3 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -9,6 +9,7 @@ import collections from typing import Optional +from typing_extensions import TypeGuard class Replacement(NamedTuple): @@ -227,7 +228,7 @@ def unmask_cell(src: str, replacements: List[Replacement]) -> str: return src -def _is_ipython_magic(node: ast.expr) -> bool: +def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: """Check if attribute is IPython magic. Note that the source of the abstract syntax tree @@ -269,7 +270,6 @@ def visit_Expr(self, node: ast.Expr) -> None: if ( isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func) - and isinstance(node.value.func, ast.Attribute) and node.value.func.attr == "run_cell_magic" ): args = [] @@ -323,7 +323,6 @@ def visit_Assign(self, node: ast.Assign) -> None: if ( isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func) - and isinstance(node.value.func, ast.Attribute) and node.value.func.attr == "getoutput" ): args = [] @@ -355,7 +354,6 @@ def visit_Expr(self, node: ast.Expr) -> None: and we look for instances of any of the latter. """ if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func): - assert isinstance(node.value.func, ast.Attribute) # help mypy args = [] for arg in node.value.args: assert isinstance(arg, ast.Str) From 49b9e597dddb761dd70f29386fd1e609cd050da5 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 12:07:29 +0100 Subject: [PATCH 50/81] dont process cell containing transformed magic --- src/black/__init__.py | 21 +++++++++++++++++++++ src/black/handle_ipynb_magics.py | 9 +++++++++ tests/test_ipynb.py | 6 ++++++ 3 files changed, 36 insertions(+) diff --git a/src/black/__init__.py b/src/black/__init__.py index 3b6ea55b91c..2f917b61257 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -53,6 +53,7 @@ unmask_cell, remove_trailing_semicolon, put_trailing_semicolon_back, + TRANSFORMED_MAGICS, ) @@ -886,6 +887,25 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo return dst_contents +def validate_cell(src: str) -> None: + """Check that cell does not already contain TransformerManager transformations. + + If a cell contains ``!ls``, then it'll be transformed to + ``get_ipython().system('ls')``. However, if the cell originally contained + ``get_ipython().system('ls')``, then it would get transformed in the same way: + + >>> TransformerManager().transform_cell("get_ipython().system('ls')") + "get_ipython().system('ls')\n" + >>> TransformerManager().transform_cell("!ls") + "get_ipython().system('ls')\n" + + Due to the impossibility of safely roundtripping in such situations, cells + containing transformed magics will be ignored. + """ + if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS): + raise NothingChanged + + def format_cell(src: str, *, fast: bool, mode: Mode) -> str: """Format code in given cell of Jupyter notebook. @@ -902,6 +922,7 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: could potentially be automagics or multi-line magics, which are currently not supported. """ + validate_cell(src) src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 6887fb631e3..c6e40d8a157 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -11,6 +11,15 @@ from typing import Optional from typing_extensions import TypeGuard +TRANSFORMED_MAGICS = frozenset( + ( + "get_ipython().run_cell_magic", + "get_ipython().system", + "get_ipython().getoutput", + "get_ipython().run_line_magic", + ) +) + class Replacement(NamedTuple): mask: str diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index ec856d7e83a..05682fbf7a4 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -76,6 +76,12 @@ def test_set_input() -> None: format_cell(src, fast=True, mode=JUPYTER_MODE) +def test_input_already_contains_transformed_magic() -> None: + src = '%time foo()\nget_ipython().run_cell_magic("time", "", "foo()\\n")' + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + def test_magic_noop() -> None: src = "ls = !ls" with pytest.raises(NothingChanged): From 98cb06a70efa677ecfafa2b227fedbfb2cd1df39 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 12:13:16 +0100 Subject: [PATCH 51/81] require typing extensions before 3.10 so as to have TypeGuard --- Pipfile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index a27614c94a9..ae1f884cad5 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,6 @@ pathspec = ">=0.8.1" regex = ">=2020.1.8" toml = ">=0.10.1" typed-ast = "==1.4.2" -typing_extensions = {"python_version <" = "3.8","version >=" = "3.7.4"} +typing_extensions = {"python_version <" = "3.10","version >=" = "3.7.4"} black = {editable = true,extras = ["d"],path = "."} dataclasses = {"python_version <" = "3.7","version >" = "0.1.3"} diff --git a/setup.py b/setup.py index cfb043ed246..d2ac3ad644b 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def get_long_description() -> str: "regex>=2020.1.8", "pathspec>=0.8.1, <1", "dataclasses>=0.6; python_version < '3.7'", - "typing_extensions>=3.7.4; python_version < '3.8'", + "typing_extensions>=3.7.4; python_version < '3.10'", "mypy_extensions>=0.4.3", ], extras_require={ From aba9d41174c7640f93fbee6a0567900e9e7716af Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 12:45:07 +0100 Subject: [PATCH 52/81] use dataclasses --- src/black/handle_ipynb_magics.py | 49 +++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index c6e40d8a157..0758a33a258 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,11 +1,12 @@ """Functions to process IPython magics with.""" +import dataclasses import ast import tokenize import io from typing import Dict import secrets -from typing import NamedTuple, List, Tuple +from typing import List, Tuple import collections from typing import Optional @@ -21,7 +22,8 @@ ) -class Replacement(NamedTuple): +@dataclasses.dataclass(frozen=True) +class Replacement: mask: str src: str @@ -176,11 +178,11 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: cell_magic_finder = CellMagicFinder() cell_magic_finder.visit(tree) - if not cell_magic_finder.header: + if cell_magic_finder.cell_magic is None: return src, replacements - mask = get_token(src, cell_magic_finder.header) - replacements.append(Replacement(mask=mask, src=cell_magic_finder.header)) - return f"{mask}\n{cell_magic_finder.body}", replacements + mask = get_token(src, cell_magic_finder.cell_magic.header) + replacements.append(Replacement(mask=mask, src=cell_magic_finder.cell_magic.header)) + return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements def replace_magics(src: str) -> Tuple[str, List[Replacement]]: @@ -207,11 +209,14 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: new_srcs = [] for i, line in enumerate(src.splitlines(), start=1): if i in magic_finder.magics: - magics = magic_finder.magics[i] - if len(magics) != 1: # pragma: nocover + offsets_and_magics = magic_finder.magics[i] + if len(offsets_and_magics) != 1: # pragma: nocover # defensive check raise UnsupportedMagic - col_offset, magic = magics[0] + col_offset, magic = ( + offsets_and_magics[0].col_offset, + offsets_and_magics[0].magic, + ) mask = get_token(src, magic) replacements.append(Replacement(mask=mask, src=magic)) line = line[:col_offset] + mask @@ -252,6 +257,12 @@ def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: ) +@dataclasses.dataclass(frozen=True) +class CellMagic: + header: str + body: str + + class CellMagicFinder(ast.NodeVisitor): """Find cell magics. @@ -271,8 +282,7 @@ class CellMagicFinder(ast.NodeVisitor): """ def __init__(self) -> None: - self.header: Optional[str] = None - self.body: Optional[str] = None + self.cell_magic: Optional[CellMagic] = None def visit_Expr(self, node: ast.Expr) -> None: """Find cell magic, extract header and body.""" @@ -288,11 +298,16 @@ def visit_Expr(self, node: ast.Expr) -> None: header = f"%%{args[0]}" if args[1]: header += f" {args[1]}" - self.header = header - self.body = args[2] + self.cell_magic = CellMagic(header=header, body=args[2]) self.generic_visit(node) +@dataclasses.dataclass(frozen=True) +class OffsetAndMagic: + col_offset: int + magic: str + + class MagicFinder(ast.NodeVisitor): """Visit cell to look for get_ipython calls. @@ -314,7 +329,7 @@ class MagicFinder(ast.NodeVisitor): def __init__(self) -> None: """Record where magics occur.""" - self.magics: Dict[int, List[Tuple[int, str]]] = collections.defaultdict(list) + self.magics: Dict[int, List[OffsetAndMagic]] = collections.defaultdict(list) def visit_Assign(self, node: ast.Assign) -> None: """Look for system assign magics. @@ -340,7 +355,9 @@ def visit_Assign(self, node: ast.Assign) -> None: args.append(arg.s) assert args src = f"!{args[0]}" - self.magics[node.value.lineno].append((node.value.col_offset, src)) + self.magics[node.value.lineno].append( + OffsetAndMagic(node.value.col_offset, src) + ) self.generic_visit(node) def visit_Expr(self, node: ast.Expr) -> None: @@ -385,7 +402,7 @@ def visit_Expr(self, node: ast.Expr) -> None: else: raise UnsupportedMagic self.magics[node.value.lineno].append( - ( + OffsetAndMagic( node.value.col_offset, src, ) From 9a421f845da2e8112ea1c4680f10aea32480d35c Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 12:53:16 +0100 Subject: [PATCH 53/81] mention black[jupyter] in docs as well as in README --- docs/getting_started.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index a509d34e903..253d97100f9 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -18,7 +18,8 @@ Also, you can try out _Black_ online for minimal fuss on the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run, but can format Python 2 code too. Python 2 support needs the `typed_ast` -dependency, which be installed with `pip install black[python2]`. +dependency, which be installed with `pip install black[python2]`. If you want format +Jupyter Notebooks as well, install with `pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: From b40f7d63427c2008ecd7320de4d9e6f3ba56d5a5 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 10 Jul 2021 14:30:15 +0100 Subject: [PATCH 54/81] add faq --- docs/faq.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index ac5ba937c1c..d7e6a16351f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -37,6 +37,29 @@ Most likely because it is ignored in `.gitignore` or excluded with configuration [file collection and discovery](usage_and_configuration/file_collection_and_discovery.md) for details. +## Why is my Jupyter Notebook cell not formatted? + +_Black_ is timid about formatting Jupyter Notebooks. Cells containing any of the +following will not be formatted: + +- automagics (e.g. `pip install black`) +- multiline magics, e.g.: + + ```python + %timeit f(1, \ + 2, \ + 3) + ``` + +- code which `IPython`'s `TransformerManager` would transform magics into, e.g.: + + ```python + get_ipython().system('ls') + ``` + +- invalid syntax, as it can't be safely distinguished from automagics in the absense of + a running `IPython` kernel. + ## Why are Flake8's E203 and W503 violated? Because they go against PEP 8. E203 falsely triggers on list From 0fa616a855e1520ff7664392ce24d934e27a912d Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Sun, 11 Jul 2021 08:51:38 +0100 Subject: [PATCH 55/81] add message to assertion error --- src/black/handle_ipynb_magics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 0758a33a258..abed7e4b0d0 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -149,7 +149,11 @@ def get_token(src: str, magic: str) -> str: token = secrets.token_hex(nbytes) counter += 1 if counter > 100: - raise AssertionError() + raise AssertionError( + "INTERNAL ERROR: Black was not able to replace IPython magic. " + "Please report a bug on https://github.com/psf/black/issues. " + f"This invalid magic might be helpful: {magic}" + ) from None if len(token) + 2 < len(magic): token = f"{token}." return f'"{token}"' From c8d12df899dd99c738fb75793db357f0e4a55717 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Sun, 11 Jul 2021 17:54:11 +0100 Subject: [PATCH 56/81] add test for indented quieted cell --- src/black/handle_ipynb_magics.py | 2 ++ tests/test_ipynb.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index abed7e4b0d0..8b0ba29807e 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -55,6 +55,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, + tokenize.DEDENT, ): continue if token[0] == tokenize.OP and token[1] == ";": @@ -81,6 +82,7 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, + tokenize.DEDENT, ): continue # We're iterating backwards, so `-idx`. diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 05682fbf7a4..0584f5c4d8a 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -30,6 +30,12 @@ def test_trailing_semicolon_with_comment() -> None: assert result == expected +def test_trailing_semicolon_indented() -> None: + src = "with foo:\n plot_bar();" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + def test_trailing_semicolon_noop() -> None: src = 'foo = "a";' with pytest.raises(NothingChanged): From a6366c4e47cb2fa262a7e504be7ecd14c8aff7ec Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 12 Jul 2021 00:17:46 +0100 Subject: [PATCH 57/81] use tokenize_rt else we cant roundtrip --- setup.py | 2 +- src/black/__init__.py | 7 +++- src/black/handle_ipynb_magics.py | 70 +++++++++++++++++--------------- tests/test_ipynb.py | 6 +++ tests/test_no_ipynb.py | 5 ++- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/setup.py b/setup.py index d2ac3ad644b..21765a4d2f9 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ def get_long_description() -> str: "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], "uvloop": ["uvloop>=0.15.2"], - "jupyter": ["ipython>=7.8.0"], + "jupyter": ["ipython>=7.8.0", "tokenize-rt>=3.2.0"], }, test_suite="tests.test_black", classifiers=[ diff --git a/src/black/__init__.py b/src/black/__init__.py index 2f917b61257..93629816a5d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -764,7 +764,10 @@ def format_file_in_place( except NothingChanged: return False except ModuleNotFoundError: - warnings.warn(f"Skipping '{src}' as IPython is not installed.\n") + warnings.warn( + f"Skipping '{src}' as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``" + ) return False if write_back == WriteBack.YES: @@ -882,7 +885,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo raise NothingChanged if not fast and not mode.is_ipynb: - # ipynb files are checked cell-by-cell + # Jupyter notebooks will already have been checked above. check_src_and_dst_equivalent(src_contents, dst_contents, mode=mode) return dst_contents diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 8b0ba29807e..c3ecee26b8e 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,8 +1,6 @@ """Functions to process IPython magics with.""" import dataclasses import ast -import tokenize -import io from typing import Dict import secrets @@ -45,52 +43,61 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: fig, ax = plt.subplots() ax.plot(x_data, y_data) # plot data - Mirrors the logic in `quiet` from `IPython.core.displayhook`. + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. """ - tokens = list(tokenize.generate_tokens(io.StringIO(src).readline)) + from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + + tokens = src_to_tokens(src) trailing_semicolon = False - for idx, token in enumerate(reversed(tokens), start=1): - if token[0] in ( - tokenize.ENDMARKER, - tokenize.NL, - tokenize.NEWLINE, - tokenize.COMMENT, - tokenize.DEDENT, - ): + for idx, token in reversed_enumerate(tokens): + if token.name in { + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", + }: continue - if token[0] == tokenize.OP and token[1] == ";": - # We're iterating backwards, so `-idx`. - del tokens[-idx] + if token.name == "OP" and token.src == ";": + del tokens[idx] trailing_semicolon = True break if not trailing_semicolon: return src, False - return tokenize.untokenize(tokens), True + return tokens_to_src(tokens), True def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: """Put trailing semicolon back if cell originally had it. - Mirrors the logic in `quiet` from `IPython.core.displayhook`. + Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses + ``tokenize_rt`` so that round-tripping works fine. """ if not has_trailing_semicolon: return src - tokens = list(tokenize.generate_tokens(io.StringIO(src).readline)) - for idx, token in enumerate(reversed(tokens), start=1): - if token[0] in ( - tokenize.ENDMARKER, - tokenize.NL, - tokenize.NEWLINE, - tokenize.COMMENT, - tokenize.DEDENT, - ): + from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + + tokens = src_to_tokens(src) + for idx, token in reversed_enumerate(tokens): + if token.name in { + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", + }: continue # We're iterating backwards, so `-idx`. - tokens[-idx] = token._replace(string=token.string + ";") + tokens[idx] = token._replace(src=token.src + ";") break else: # pragma: nocover raise AssertionError("Unreachable code") - return str(tokenize.untokenize(tokens)) + return str(tokens_to_src(tokens)) def mask_cell(src: str) -> Tuple[str, List[Replacement]]: @@ -154,7 +161,7 @@ def get_token(src: str, magic: str) -> str: raise AssertionError( "INTERNAL ERROR: Black was not able to replace IPython magic. " "Please report a bug on https://github.com/psf/black/issues. " - f"This invalid magic might be helpful: {magic}" + f"The magic might be helpful: {magic}" ) from None if len(token) + 2 < len(magic): token = f"{token}." @@ -408,9 +415,6 @@ def visit_Expr(self, node: ast.Expr) -> None: else: raise UnsupportedMagic self.magics[node.value.lineno].append( - OffsetAndMagic( - node.value.col_offset, - src, - ) + OffsetAndMagic(node.value.col_offset, src) ) self.generic_visit(node) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 0584f5c4d8a..5ca2096a4dc 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -30,6 +30,12 @@ def test_trailing_semicolon_with_comment() -> None: assert result == expected +def test_trailing_semicolon_with_comment_on_next_line() -> None: + src = "import black;\n\n# this is a comment" + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + def test_trailing_semicolon_indented() -> None: src = "with foo:\n plot_bar();" with pytest.raises(NothingChanged): diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index 5ba2b850cb3..531024ae709 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -9,7 +9,10 @@ def test_ipynb_diff_with_no_change() -> None: path = THIS_DIR / "data/notebook_trailing_newline.ipynb" - msg = r"Skipping '.*' as IPython is not installed." + msg = ( + r"Skipping '.*' as Jupyter dependencies are not installed.\n" + r"You can fix this by running ``pip install black\[jupyter\]``" + ) with pytest.warns(UserWarning, match=msg): format_file_in_place( path, fast=False, mode=DEFAULT_MODE, write_back=WriteBack.DIFF From cd70366ef087eee34df2d755cee5c51a44f9fe1e Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 12 Jul 2021 17:49:22 +0100 Subject: [PATCH 58/81] fmake fronzet set for tokens to ignore when looking for trailing semicolon --- src/black/handle_ipynb_magics.py | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index c3ecee26b8e..ebbb1464d5d 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -18,6 +18,17 @@ "get_ipython().run_line_magic", ) ) +TOKENS_TO_IGNORE = frozenset( + ( + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", + ) +) @dataclasses.dataclass(frozen=True) @@ -46,20 +57,16 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses ``tokenize_rt`` so that round-tripping works fine. """ - from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + from tokenize_rt import ( + src_to_tokens, + tokens_to_src, + reversed_enumerate, + ) tokens = src_to_tokens(src) trailing_semicolon = False for idx, token in reversed_enumerate(tokens): - if token.name in { - "ENDMARKER", - "NL", - "NEWLINE", - "COMMENT", - "DEDENT", - "UNIMPORTANT_WS", - "ESCAPED_NL", - }: + if token.name in TOKENS_TO_IGNORE: continue if token.name == "OP" and token.src == ";": del tokens[idx] @@ -82,21 +89,15 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: tokens = src_to_tokens(src) for idx, token in reversed_enumerate(tokens): - if token.name in { - "ENDMARKER", - "NL", - "NEWLINE", - "COMMENT", - "DEDENT", - "UNIMPORTANT_WS", - "ESCAPED_NL", - }: + if token.name in TOKENS_TO_IGNORE: continue - # We're iterating backwards, so `-idx`. tokens[idx] = token._replace(src=token.src + ";") break else: # pragma: nocover - raise AssertionError("Unreachable code") + raise AssertionError( + "INTERNAL ERROR: Was not able to reinstate trailing semicolon. " + "Please report a bug on https://github.com/psf/black/issues. " + ) from None return str(tokens_to_src(tokens)) From 4c3da5d95b6fff356e30f7712a4f9c46e438494e Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Mon, 12 Jul 2021 18:26:53 +0100 Subject: [PATCH 59/81] remove planetary code examples as recent commits result in changes --- src/black_primer/primer.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index c04b1884cd1..b37f8bf6516 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -76,13 +76,6 @@ "long_checkout": false, "py_versions": ["all"] }, - "planetary computer examples": { - "cli_arguments": ["--experimental-string-processing"], - "expect_formatting_changes": false, - "git_clone_url": "https://github.com/microsoft/PlanetaryComputerExamples", - "long_checkout": false, - "py_versions": ["all"] - }, "poetry": { "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, From 02151f071ed5ef4b5d5cfd3632a755332945ab13 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Tue, 13 Jul 2021 17:32:26 +0100 Subject: [PATCH 60/81] use dataclasses which inherit from ast.NodeVisitor --- src/black/handle_ipynb_magics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index ebbb1464d5d..1d4861cc722 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -277,6 +277,7 @@ class CellMagic: body: str +@dataclasses.dataclass class CellMagicFinder(ast.NodeVisitor): """Find cell magics. @@ -295,8 +296,7 @@ class CellMagicFinder(ast.NodeVisitor): and we look for instances of the latter. """ - def __init__(self) -> None: - self.cell_magic: Optional[CellMagic] = None + cell_magic: Optional[CellMagic] = None def visit_Expr(self, node: ast.Expr) -> None: """Find cell magic, extract header and body.""" @@ -322,6 +322,7 @@ class OffsetAndMagic: magic: str +@dataclasses.dataclass class MagicFinder(ast.NodeVisitor): """Visit cell to look for get_ipython calls. @@ -341,9 +342,9 @@ class MagicFinder(ast.NodeVisitor): types of magics). """ - def __init__(self) -> None: - """Record where magics occur.""" - self.magics: Dict[int, List[OffsetAndMagic]] = collections.defaultdict(list) + magics: Dict[int, List[OffsetAndMagic]] = dataclasses.field( + default_factory=lambda: collections.defaultdict(list) + ) def visit_Assign(self, node: ast.Assign) -> None: """Look for system assign magics. From ec627687880b8e3df11cefcdea1a62f564580476 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 14 Jul 2021 18:40:10 +0100 Subject: [PATCH 61/81] bump typing-extensions so that TypeGuard is available --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 267daf128fa..f3de3ba746d 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def get_long_description() -> str: "regex>=2020.1.8", "pathspec>=0.8.1, <1", "dataclasses>=0.6; python_version < '3.7'", - "typing_extensions>=3.7.4; python_version < '3.10'", + "typing_extensions>=3.10.0.0; python_version < '3.10'", "mypy_extensions>=0.4.3", ], extras_require={ From dce54b692f650cf3f92adf01993994a2a7b80c33 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 14 Jul 2021 18:41:29 +0100 Subject: [PATCH 62/81] bump typing-extensions in Pipfile --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index c72f36e0118..97da7591cd6 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,6 @@ pathspec = ">=0.8.1" regex = ">=2020.1.8" tomli = ">=0.2.6, <2.0.0" typed-ast = "==1.4.2" -typing_extensions = {"python_version <" = "3.10","version >=" = "3.7.4"} +typing_extensions = {"python_version <" = "3.10","version >=" = "3.10.0.0"} black = {editable = true,extras = ["d"],path = "."} dataclasses = {"python_version <" = "3.7","version >" = "0.1.3"} From f7f3ce4d49dfaa4a25bd78892468f05dd8fc861f Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 14 Jul 2021 22:02:14 +0100 Subject: [PATCH 63/81] add test with notebook with empty metadata --- src/black/__init__.py | 20 +++++++++--- tests/data/notebook_empty_metadata.ipynb | 27 ++++++++++++++++ tests/test_ipynb.py | 39 ++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 tests/data/notebook_empty_metadata.ipynb diff --git a/src/black/__init__.py b/src/black/__init__.py index 9f5df25fe09..2b127e328fc 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -20,6 +20,7 @@ Generator, Iterator, List, + MutableMapping, Optional, Pattern, Set, @@ -891,7 +892,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo return dst_contents -def validate_cell(src: str) -> None: +def _validate_cell(src: str) -> None: """Check that cell does not already contain TransformerManager transformations. If a cell contains ``!ls``, then it'll be transformed to @@ -926,7 +927,7 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: could potentially be automagics or multi-line magics, which are currently not supported. """ - validate_cell(src) + _validate_cell(src) src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) @@ -947,6 +948,18 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: return dst +def validate_metadata(nb: MutableMapping[str, Any]) -> None: + """If notebook is marked as non-Python, don't format it. + + All notebook metadata fields are optional, see + https://nbformat.readthedocs.io/en/latest/format_description.html. So + if a notebook has empty metadata, we will try to parse it anyway. + """ + language = nb.get("metadata", {}).get("language_info", {}).get("name", None) + if language is not None and language != "python": + raise NothingChanged + + def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: """Format Jupyter notebook. @@ -956,8 +969,7 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileCon trailing_newline = src_contents[-1] == "\n" modified = False nb = json.loads(src_contents) - if nb.get("metadata", {}).get("language_info", {}).get("name", None) != "python": - raise NothingChanged + validate_metadata(nb) for cell in nb["cells"]: if cell.get("cell_type", None) == "code": try: diff --git a/tests/data/notebook_empty_metadata.ipynb b/tests/data/notebook_empty_metadata.ipynb new file mode 100644 index 00000000000..7dc1f805cd6 --- /dev/null +++ b/tests/data/notebook_empty_metadata.ipynb @@ -0,0 +1,27 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print('foo')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 5ca2096a4dc..060e8b6974a 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -151,6 +151,45 @@ def test_empty_cell() -> None: format_cell(src, fast=True, mode=JUPYTER_MODE) +def test_entire_notebook_empty_metadata() -> None: + with open( + os.path.join("tests", "data", "notebook_empty_metadata.ipynb"), "rb" + ) as fd: + content_bytes = fd.read() + content = content_bytes.decode() + result = format_file_contents(content, fast=True, mode=JUPYTER_MODE) + expected = ( + "{\n" + ' "cells": [\n' + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {\n' + ' "tags": []\n' + " },\n" + ' "outputs": [],\n' + ' "source": [\n' + ' "%%time\\n",\n' + ' "\\n",\n' + ' "print(\\"foo\\")"\n' + " ]\n" + " },\n" + " {\n" + ' "cell_type": "code",\n' + ' "execution_count": null,\n' + ' "metadata": {},\n' + ' "outputs": [],\n' + ' "source": []\n' + " }\n" + " ],\n" + ' "metadata": {},\n' + ' "nbformat": 4,\n' + ' "nbformat_minor": 4\n' + "}\n" + ) + assert result == expected + + def test_entire_notebook_trailing_newline() -> None: with open( os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), "rb" From ede7de55959486f514dbaccf8ff2eb7f7f0dee75 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 14 Jul 2021 22:30:10 +0100 Subject: [PATCH 64/81] pipenv lock --- Pipfile.lock | 149 +++++++++++++++------------------------------------ 1 file changed, 43 insertions(+), 106 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 90923038adb..55599287522 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e6b7c19c10b5fba5ba20296b68049fab625b4c6d61db97b79d83a6dfa36d9f6d" + "sha256": "53fcc816c27b5b7272dddb9865ea1262dba12eda2b6926d16f211034f1c5d576" }, "pipfile-spec": 6, "requires": {}, @@ -295,9 +295,9 @@ "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", - "python_version <": "3.8", + "python_version <": "3.10", "version": "==3.10.0.0", - "version >=": "3.7.4" + "version >=": "3.10.0.0" }, "yarl": { "hashes": [ @@ -434,6 +434,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.1" }, + "backports.entry-points-selectable": { + "hashes": [ + "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a", + "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc" + ], + "markers": "python_version >= '2.7'", + "version": "==1.1.0" + }, "black": { "editable": true, "extras": [ @@ -443,11 +451,11 @@ }, "bleach": { "hashes": [ - "sha256:6123ddc1052673e52bab52cdc955bcb57a015264a1c57d37bea2f6b817af0125", - "sha256:98b3170739e5e83dd9dc19633f074727ad848cbedb6026708c8ac2d3b697a433" + "sha256:306483a5a9795474160ad57fce3ddd1b50551e981eed8e15a582d34cef28aafa", + "sha256:ae976d7174bba988c0b632def82fdc94235756edfb14e6558a9c5be555c9fb78" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==3.3.0" + "version": "==3.3.1" }, "certifi": { "hashes": [ @@ -456,51 +464,6 @@ ], "version": "==2021.5.30" }, - "cffi": { - "hashes": [ - "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", - "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", - "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", - "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", - "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", - "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", - "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", - "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", - "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", - "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", - "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", - "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", - "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", - "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", - "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", - "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", - "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", - "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", - "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", - "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", - "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", - "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", - "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", - "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", - "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", - "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", - "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", - "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", - "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", - "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", - "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", - "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", - "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", - "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", - "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", - "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", - "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", - "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", - "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", - "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" - ], - "version": "==1.14.6" - }, "cfgv": { "hashes": [ "sha256:9e600479b3b99e8af981ecdfc80a0296104ee610cab48a5ae4ffd0b668650eb1", @@ -517,6 +480,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.0.0" }, + "charset-normalizer": { + "hashes": [ + "sha256:ad0da505736fc7e716a8da15bf19a985db21ac6415c26b34d2fafd3beb3d927e", + "sha256:b68b38179052975093d71c1b5361bf64afd80484697c1f27056e50593e695ceb" + ], + "markers": "python_version >= '3'", + "version": "==2.0.1" + }, "click": { "hashes": [ "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", @@ -591,24 +562,6 @@ "index": "pypi", "version": "==5.5" }, - "cryptography": { - "hashes": [ - "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", - "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", - "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", - "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", - "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", - "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", - "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", - "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", - "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", - "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", - "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", - "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" - ], - "markers": "python_version >= '3.6'", - "version": "==3.4.7" - }, "distlib": { "hashes": [ "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736", @@ -680,14 +633,6 @@ "markers": "python_version >= '3.6'", "version": "==4.6.1" }, - "jeepney": { - "hashes": [ - "sha256:1237cd64c8f7ac3aa4b3f332c4d0fb4a8216f39eaa662ec904302d4d77de5a54", - "sha256:71335e7a4e93817982f473f3507bffc2eff7a544119ab9b73e089c8ba1409ba3" - ], - "markers": "sys_platform == 'linux'", - "version": "==0.7.0" - }, "jinja2": { "hashes": [ "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", @@ -885,6 +830,14 @@ ], "version": "==1.7.1" }, + "platformdirs": { + "hashes": [ + "sha256:0b9547541f599d3d242078ae60b927b3e453f0ad52f58b4d4bc3be86aed3ec41", + "sha256:3b00d081227d9037bbbca521a5787796b5ef5000faea1e43fd76f1d44b06fcfa" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.0.2" + }, "pre-commit": { "hashes": [ "sha256:764972c60693dc668ba8e86eb29654ec3144501310f7198742a767bec385a378", @@ -901,14 +854,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7.0" }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.20" - }, "pyflakes": { "hashes": [ "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", @@ -1032,11 +977,11 @@ }, "requests": { "hashes": [ - "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", - "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.25.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.26.0" }, "requests-toolbelt": { "hashes": [ @@ -1052,14 +997,6 @@ ], "version": "==1.5.0" }, - "secretstorage": { - "hashes": [ - "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", - "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" - ], - "markers": "sys_platform == 'linux'", - "version": "==3.3.1" - }, "setuptools-scm": { "hashes": [ "sha256:c3bd5f701c8def44a5c0bfe8d407bef3f80342217ef3492b951f3777bd2d915c", @@ -1085,11 +1022,11 @@ }, "sphinx": { "hashes": [ - "sha256:4219f14258ca5612a0c85ed9b7222d54da69724d7e9dd92d1819ad1bf65e1ad2", - "sha256:51028bb0d3340eb80bcc1a2d614e8308ac78d226e6b796943daf57920abc1aea" + "sha256:23c846a1841af998cb736218539bb86d16f5eb95f5760b1966abcd2d584e62b8", + "sha256:3d513088236eef51e5b0adb78b0492eb22cc3b8ccdb0b36dd021173b365d4454" ], "index": "pypi", - "version": "==4.1.0" + "version": "==4.1.1" }, "sphinx-copybutton": { "hashes": [ @@ -1218,9 +1155,9 @@ "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", - "python_version <": "3.8", + "python_version <": "3.10", "version": "==3.10.0.0", - "version >=": "3.7.4" + "version >=": "3.10.0.0" }, "urllib3": { "hashes": [ @@ -1232,11 +1169,11 @@ }, "virtualenv": { "hashes": [ - "sha256:14fdf849f80dbb29a4eb6caa9875d476ee2a5cf76a5f5415fa2f1606010ab467", - "sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76" + "sha256:51df5d8a2fad5d1b13e088ff38a433475768ff61f202356bb9812c454c20ae45", + "sha256:e4fc84337dce37ba34ef520bf2d4392b392999dbe47df992870dc23230f6b758" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4.7" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.6.0" }, "webencodings": { "hashes": [ From 67d6bcec14507aac3964b9cdc6669236e55e9450 Mon Sep 17 00:00:00 2001 From: MarcoGorelli Date: Wed, 14 Jul 2021 22:36:03 +0100 Subject: [PATCH 65/81] deprivative validate_cell --- Pipfile.lock | 149 ++++++++++++++++++++++++++++++------------ src/black/__init__.py | 4 +- 2 files changed, 108 insertions(+), 45 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 55599287522..90923038adb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "53fcc816c27b5b7272dddb9865ea1262dba12eda2b6926d16f211034f1c5d576" + "sha256": "e6b7c19c10b5fba5ba20296b68049fab625b4c6d61db97b79d83a6dfa36d9f6d" }, "pipfile-spec": 6, "requires": {}, @@ -295,9 +295,9 @@ "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", - "python_version <": "3.10", + "python_version <": "3.8", "version": "==3.10.0.0", - "version >=": "3.10.0.0" + "version >=": "3.7.4" }, "yarl": { "hashes": [ @@ -434,14 +434,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.1" }, - "backports.entry-points-selectable": { - "hashes": [ - "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a", - "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc" - ], - "markers": "python_version >= '2.7'", - "version": "==1.1.0" - }, "black": { "editable": true, "extras": [ @@ -451,11 +443,11 @@ }, "bleach": { "hashes": [ - "sha256:306483a5a9795474160ad57fce3ddd1b50551e981eed8e15a582d34cef28aafa", - "sha256:ae976d7174bba988c0b632def82fdc94235756edfb14e6558a9c5be555c9fb78" + "sha256:6123ddc1052673e52bab52cdc955bcb57a015264a1c57d37bea2f6b817af0125", + "sha256:98b3170739e5e83dd9dc19633f074727ad848cbedb6026708c8ac2d3b697a433" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==3.3.1" + "version": "==3.3.0" }, "certifi": { "hashes": [ @@ -464,6 +456,51 @@ ], "version": "==2021.5.30" }, + "cffi": { + "hashes": [ + "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", + "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", + "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", + "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", + "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", + "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", + "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", + "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", + "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", + "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", + "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", + "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", + "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", + "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", + "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", + "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", + "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", + "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", + "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", + "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", + "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", + "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", + "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", + "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", + "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", + "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", + "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", + "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", + "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", + "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", + "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", + "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", + "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", + "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", + "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", + "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", + "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", + "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", + "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", + "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" + ], + "version": "==1.14.6" + }, "cfgv": { "hashes": [ "sha256:9e600479b3b99e8af981ecdfc80a0296104ee610cab48a5ae4ffd0b668650eb1", @@ -480,14 +517,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.0.0" }, - "charset-normalizer": { - "hashes": [ - "sha256:ad0da505736fc7e716a8da15bf19a985db21ac6415c26b34d2fafd3beb3d927e", - "sha256:b68b38179052975093d71c1b5361bf64afd80484697c1f27056e50593e695ceb" - ], - "markers": "python_version >= '3'", - "version": "==2.0.1" - }, "click": { "hashes": [ "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", @@ -562,6 +591,24 @@ "index": "pypi", "version": "==5.5" }, + "cryptography": { + "hashes": [ + "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", + "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", + "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", + "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", + "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", + "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", + "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", + "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", + "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", + "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", + "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", + "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" + ], + "markers": "python_version >= '3.6'", + "version": "==3.4.7" + }, "distlib": { "hashes": [ "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736", @@ -633,6 +680,14 @@ "markers": "python_version >= '3.6'", "version": "==4.6.1" }, + "jeepney": { + "hashes": [ + "sha256:1237cd64c8f7ac3aa4b3f332c4d0fb4a8216f39eaa662ec904302d4d77de5a54", + "sha256:71335e7a4e93817982f473f3507bffc2eff7a544119ab9b73e089c8ba1409ba3" + ], + "markers": "sys_platform == 'linux'", + "version": "==0.7.0" + }, "jinja2": { "hashes": [ "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", @@ -830,14 +885,6 @@ ], "version": "==1.7.1" }, - "platformdirs": { - "hashes": [ - "sha256:0b9547541f599d3d242078ae60b927b3e453f0ad52f58b4d4bc3be86aed3ec41", - "sha256:3b00d081227d9037bbbca521a5787796b5ef5000faea1e43fd76f1d44b06fcfa" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.0.2" - }, "pre-commit": { "hashes": [ "sha256:764972c60693dc668ba8e86eb29654ec3144501310f7198742a767bec385a378", @@ -854,6 +901,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.7.0" }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, "pyflakes": { "hashes": [ "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", @@ -977,11 +1032,11 @@ }, "requests": { "hashes": [ - "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", - "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==2.26.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.25.1" }, "requests-toolbelt": { "hashes": [ @@ -997,6 +1052,14 @@ ], "version": "==1.5.0" }, + "secretstorage": { + "hashes": [ + "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", + "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" + ], + "markers": "sys_platform == 'linux'", + "version": "==3.3.1" + }, "setuptools-scm": { "hashes": [ "sha256:c3bd5f701c8def44a5c0bfe8d407bef3f80342217ef3492b951f3777bd2d915c", @@ -1022,11 +1085,11 @@ }, "sphinx": { "hashes": [ - "sha256:23c846a1841af998cb736218539bb86d16f5eb95f5760b1966abcd2d584e62b8", - "sha256:3d513088236eef51e5b0adb78b0492eb22cc3b8ccdb0b36dd021173b365d4454" + "sha256:4219f14258ca5612a0c85ed9b7222d54da69724d7e9dd92d1819ad1bf65e1ad2", + "sha256:51028bb0d3340eb80bcc1a2d614e8308ac78d226e6b796943daf57920abc1aea" ], "index": "pypi", - "version": "==4.1.1" + "version": "==4.1.0" }, "sphinx-copybutton": { "hashes": [ @@ -1155,9 +1218,9 @@ "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" ], "index": "pypi", - "python_version <": "3.10", + "python_version <": "3.8", "version": "==3.10.0.0", - "version >=": "3.10.0.0" + "version >=": "3.7.4" }, "urllib3": { "hashes": [ @@ -1169,11 +1232,11 @@ }, "virtualenv": { "hashes": [ - "sha256:51df5d8a2fad5d1b13e088ff38a433475768ff61f202356bb9812c454c20ae45", - "sha256:e4fc84337dce37ba34ef520bf2d4392b392999dbe47df992870dc23230f6b758" + "sha256:14fdf849f80dbb29a4eb6caa9875d476ee2a5cf76a5f5415fa2f1606010ab467", + "sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.6.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.4.7" }, "webencodings": { "hashes": [ diff --git a/src/black/__init__.py b/src/black/__init__.py index 2b127e328fc..f60fb97a219 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -892,7 +892,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo return dst_contents -def _validate_cell(src: str) -> None: +def validate_cell(src: str) -> None: """Check that cell does not already contain TransformerManager transformations. If a cell contains ``!ls``, then it'll be transformed to @@ -927,7 +927,7 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: could potentially be automagics or multi-line magics, which are currently not supported. """ - _validate_cell(src) + validate_cell(src) src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon( src ) From c240b2c4b69bc97fef0260f06b54e9bb884ae2c4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jul 2021 18:14:51 -0700 Subject: [PATCH 66/81] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9bd24bdd8f5..709478e1d58 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run. If you want to format Python 2 code as well, install with -`pip install black[python2]`. If you want format Jupyter Notebooks as well, install with +`pip install black[python2]`. If you want to format Jupyter Notebooks, install with `pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: From 32d6ee6766ae206b1787882cc8d7afeb1dcb8513 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jul 2021 18:14:58 -0700 Subject: [PATCH 67/81] Update docs/getting_started.md --- docs/getting_started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started.md b/docs/getting_started.md index 253d97100f9..c79dc607c4a 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -18,8 +18,8 @@ Also, you can try out _Black_ online for minimal fuss on the _Black_ can be installed by running `pip install black`. It requires Python 3.6.2+ to run, but can format Python 2 code too. Python 2 support needs the `typed_ast` -dependency, which be installed with `pip install black[python2]`. If you want format -Jupyter Notebooks as well, install with `pip install black[jupyter]`. +dependency, which be installed with `pip install black[python2]`. If you want to format +Jupyter Notebooks, install with `pip install black[jupyter]`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: From 8f24601c1f2fb46479d6f9e6b02f26594f73fb57 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 10:36:05 +0100 Subject: [PATCH 68/81] dont cache notebooks if jupyter dependencies arent found --- src/black/__init__.py | 25 +++++++++++++++++++------ src/black/files.py | 5 +++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index f60fb97a219..6654b38cb78 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,4 +1,5 @@ import asyncio +from functools import lru_cache import warnings import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor @@ -524,6 +525,9 @@ def get_sources( if is_stdin: p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") + if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed(): + continue + sources.add(p) elif p.is_dir(): sources.update( @@ -740,6 +744,21 @@ async def schedule_formatting( write_cache(cache, sources_to_cache, mode) +@lru_cache() +def jupyter_dependencies_are_installed() -> bool: + try: + import IPython # noqa:F401 + import tokenize_rt # noqa:F401 + except ModuleNotFoundError: + warnings.warn( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``" + ) + return False + else: + return True + + def format_file_in_place( src: Path, fast: bool, @@ -765,12 +784,6 @@ def format_file_in_place( dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) except NothingChanged: return False - except ModuleNotFoundError: - warnings.warn( - f"Skipping '{src}' as Jupyter dependencies are not installed.\n" - "You can fix this by running ``pip install black[jupyter]``" - ) - return False if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: diff --git a/src/black/files.py b/src/black/files.py index 427ad668f48..bdb1780f75e 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -219,6 +219,11 @@ def gen_python_files( ) elif child.is_file(): + if child.suffix == ".ipynb": + from black import jupyter_dependencies_are_installed + + if not jupyter_dependencies_are_installed(): + continue include_match = include.search(normalized_path) if include else True if include_match: yield child From 2c14df592ee18305d8fcee228b296896a32e3f98 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 11:43:35 +0100 Subject: [PATCH 69/81] dont write to cache if jupyter deps are not installed --- src/black/__init__.py | 32 +++++++++++++++----------------- src/black/files.py | 5 +++++ tests/test_black.py | 14 ++++++++++++++ tests/test_ipynb.py | 23 ++++++++++++++++++++++- tests/test_no_ipynb.py | 21 ++++++++++++--------- 5 files changed, 68 insertions(+), 27 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 6654b38cb78..90b788b0ce7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,6 +1,6 @@ import asyncio from functools import lru_cache -import warnings +from json.decoder import JSONDecodeError import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor from contextlib import contextmanager @@ -207,14 +207,6 @@ def validate_regex( " when piping source on standard input)." ), ) -@click.option( - "--ipynb", - is_flag=True, - help=( - "Format all input files like ipynb notebooks regardless of file extension " - "(useful when piping source on standard input)." - ), -) @click.option( "-S", "--skip-string-normalization", @@ -374,7 +366,6 @@ def main( color: bool, fast: bool, pyi: bool, - ipynb: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, experimental_string_processing: bool, @@ -411,7 +402,6 @@ def main( target_versions=versions, line_length=line_length, is_pyi=pyi, - is_ipynb=ipynb, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, experimental_string_processing=experimental_string_processing, @@ -525,7 +515,9 @@ def get_sources( if is_stdin: p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") - if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed(): + if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + verbose, quiet + ): continue sources.add(p) @@ -540,6 +532,8 @@ def get_sources( force_exclude, report, gitignore, + verbose=verbose, + quiet=quiet, ) ) elif s == "-": @@ -745,15 +739,17 @@ async def schedule_formatting( @lru_cache() -def jupyter_dependencies_are_installed() -> bool: +def jupyter_dependencies_are_installed(verbose: bool, quiet: bool) -> bool: try: import IPython # noqa:F401 import tokenize_rt # noqa:F401 except ModuleNotFoundError: - warnings.warn( - "Skipping .ipynb files as Jupyter dependencies are not installed.\n" - "You can fix this by running ``pip install black[jupyter]``" - ) + if verbose or not quiet: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``" + ) + out(msg) return False else: return True @@ -784,6 +780,8 @@ def format_file_in_place( dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) except NothingChanged: return False + except JSONDecodeError: + raise ValueError(f"File '{src}' cannot be parsed as valid Jupyter notebook.") if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: diff --git a/src/black/files.py b/src/black/files.py index bdb1780f75e..16ad4f02daa 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -165,6 +165,9 @@ def gen_python_files( force_exclude: Optional[Pattern[str]], report: Report, gitignore: Optional[PathSpec], + *, + verbose: bool, + quiet: bool, ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, @@ -216,6 +219,8 @@ def gen_python_files( force_exclude, report, gitignore + get_gitignore(child) if gitignore is not None else None, + verbose=verbose, + quiet=quiet, ) elif child.is_file(): diff --git a/tests/test_black.py b/tests/test_black.py index e3be9c71fd4..0a716d764a8 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1378,6 +1378,8 @@ def test_include_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1689,6 +1691,8 @@ def test_gitignore_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1716,6 +1720,8 @@ def test_nested_gitignore(self) -> None: None, report, root_gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1750,6 +1756,8 @@ def test_empty_include(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1774,6 +1782,8 @@ def test_extend_exclude(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1846,6 +1856,8 @@ def test_symlink_out_of_root_directory(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) except ValueError as ve: @@ -1867,6 +1879,8 @@ def test_symlink_out_of_root_directory(self) -> None: None, report, gitignore, + verbose=False, + quiet=False, ) ) path.iterdir.assert_called() diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 060e8b6974a..49b438816fc 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,10 +1,18 @@ -from black import NothingChanged, format_cell, format_file_contents +import pathlib +from black import ( + NothingChanged, + format_cell, + format_file_contents, + format_file_in_place, +) import os import pytest import subprocess from black import Mode pytestmark = pytest.mark.jupyter +pytest.importorskip("IPython", reason="IPython is an optional dependency") +pytest.importorskip("tokenize_rt", reason="tokenize-rt is an optional dependency") JUPYTER_MODE = Mode(is_ipynb=True) @@ -315,6 +323,19 @@ def test_empty_string() -> None: format_file_contents("", fast=True, mode=JUPYTER_MODE) +def test_unparseable_notebook() -> None: + msg = ( + r"File 'tests[/\\]data[/\\]notebook_which_cant_be_parsed\.ipynb' " + r"cannot be parsed as valid Jupyter notebook\." + ) + with pytest.raises(ValueError, match=msg): + format_file_in_place( + pathlib.Path("tests") / "data/notebook_which_cant_be_parsed.ipynb", + fast=True, + mode=JUPYTER_MODE, + ) + + def test_ipynb_diff_with_change() -> None: output = subprocess.run( [ diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index 531024ae709..6ca996d35fd 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,6 +1,5 @@ -from tests.util import DEFAULT_MODE +import subprocess import pytest -from black import WriteBack, format_file_in_place from tests.util import THIS_DIR @@ -9,11 +8,15 @@ def test_ipynb_diff_with_no_change() -> None: path = THIS_DIR / "data/notebook_trailing_newline.ipynb" - msg = ( - r"Skipping '.*' as Jupyter dependencies are not installed.\n" - r"You can fix this by running ``pip install black\[jupyter\]``" + output = subprocess.run( + ["black", str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, ) - with pytest.warns(UserWarning, match=msg): - format_file_in_place( - path, fast=False, mode=DEFAULT_MODE, write_back=WriteBack.DIFF - ) + expected_stderr = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``\n" + ) + result_stderr = output.stderr + assert expected_stderr in result_stderr From ce392f55a8a374f913dffeb318bfc31260d01fd0 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 11:45:14 +0100 Subject: [PATCH 70/81] add notebook which cant be parsed --- tests/data/notebook_which_cant_be_parsed.ipynb | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/data/notebook_which_cant_be_parsed.ipynb diff --git a/tests/data/notebook_which_cant_be_parsed.ipynb b/tests/data/notebook_which_cant_be_parsed.ipynb new file mode 100644 index 00000000000..257cc5642cb --- /dev/null +++ b/tests/data/notebook_which_cant_be_parsed.ipynb @@ -0,0 +1 @@ +foo From 7ad25a1fbcb86944cfe47c175a20b19e995fa218 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 12:09:01 +0100 Subject: [PATCH 71/81] use clirunner --- tests/test_ipynb.py | 26 ++++++++++++++++++++++++++ tests/test_no_ipynb.py | 16 ++++++---------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 49b438816fc..0f3d0bbeb3f 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,4 +1,6 @@ import pathlib +from click.testing import CliRunner +from black import main from black import ( NothingChanged, format_cell, @@ -9,6 +11,8 @@ import pytest import subprocess from black import Mode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.tmpdir import tmpdir pytestmark = pytest.mark.jupyter pytest.importorskip("IPython", reason="IPython is an optional dependency") @@ -365,3 +369,25 @@ def test_ipynb_diff_with_no_change() -> None: result = output.stdout expected = "" assert result == expected + + +def test_cache_isnt_written_if_no_jupyter_deps( + monkeypatch: MonkeyPatch, tmpdir: tmpdir +) -> None: + # Check that the cache isn't written to if Jupyter dependencies aren't installed. + with open( + os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + ) as src, open(tmpdir / "notebook.ipynb", "w") as dst: + dst.write(src.read()) + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False + ) + runner = CliRunner() + result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + assert "No Python files are present to be formatted. Nothing to do" in result.output + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True + ) + runner = CliRunner() + result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + assert "reformatted" in result.output diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index 6ca996d35fd..bd48d32cf97 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,22 +1,18 @@ -import subprocess import pytest from tests.util import THIS_DIR +from black import main +from click.testing import CliRunner pytestmark = pytest.mark.no_jupyter def test_ipynb_diff_with_no_change() -> None: + runner = CliRunner() path = THIS_DIR / "data/notebook_trailing_newline.ipynb" - output = subprocess.run( - ["black", str(path)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - expected_stderr = ( + result = runner.invoke(main, [str(path)]) + expected_output = ( "Skipping .ipynb files as Jupyter dependencies are not installed.\n" "You can fix this by running ``pip install black[jupyter]``\n" ) - result_stderr = output.stderr - assert expected_stderr in result_stderr + assert expected_output in result.output From 9ccf84e639ec9ea589ae86ca1299017ae403f78d Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 12:17:23 +0100 Subject: [PATCH 72/81] remove other subprocess calls --- tests/test_ipynb.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 0f3d0bbeb3f..7e1bc167b15 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -9,7 +9,6 @@ ) import os import pytest -import subprocess from black import Mode from _pytest.monkeypatch import MonkeyPatch from _pytest.tmpdir import tmpdir @@ -20,6 +19,8 @@ JUPYTER_MODE = Mode(is_ipynb=True) +runner = CliRunner() + def test_noop() -> None: src = 'foo = "a"' @@ -341,34 +342,27 @@ def test_unparseable_notebook() -> None: def test_ipynb_diff_with_change() -> None: - output = subprocess.run( + result = runner.invoke( + main, [ - "black", os.path.join("tests", "data", "notebook_trailing_newline.ipynb"), "--diff", ], - stdout=subprocess.PIPE, - universal_newlines=True, ) - # Ignore the first two lines of output as they contain the current UTC time - result = "".join(output.stdout.splitlines(keepends=True)[2:]) - expected = "@@ -1,3 +1,3 @@\n" " %%time\n" " \n" "-print('foo')\n" '+print("foo")\n' - assert result == expected + expected = "@@ -1,3 +1,3 @@\n %%time\n \n-print('foo')\n" '+print("foo")\n' + assert expected in result.output def test_ipynb_diff_with_no_change() -> None: - output = subprocess.run( + result = runner.invoke( + main, [ - "black", os.path.join("tests", "data", "notebook_without_changes.ipynb"), "--diff", ], - stdout=subprocess.PIPE, - universal_newlines=True, ) - result = output.stdout - expected = "" - assert result == expected + expected = "1 file would be left unchanged." + assert expected in result.output def test_cache_isnt_written_if_no_jupyter_deps( @@ -382,12 +376,10 @@ def test_cache_isnt_written_if_no_jupyter_deps( monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False ) - runner = CliRunner() result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) assert "No Python files are present to be formatted. Nothing to do" in result.output monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True ) - runner = CliRunner() result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) assert "reformatted" in result.output From 7d4cbf6109476531809f0984db16a6f5f71ab9cc Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 12:24:31 +0100 Subject: [PATCH 73/81] add docstring --- src/black/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 90b788b0ce7..ecfcb420e99 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -863,9 +863,15 @@ def format_stdin_to_stdout( f.detach() -def check_src_and_dst_equivalent( +def check_stability_and_equivalence( src_contents: str, dst_contents: str, *, mode: Mode ) -> None: + """Perform stability and equivalence checks. + + Raise AssertionError if source and destination contents are not + equivalent, or if a second pass of the formatter would format the + content differently. + """ assert_equivalent(src_contents, dst_contents) # Forced second pass to work around optional trailing commas (becoming @@ -899,7 +905,7 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if not fast and not mode.is_ipynb: # Jupyter notebooks will already have been checked above. - check_src_and_dst_equivalent(src_contents, dst_contents, mode=mode) + check_stability_and_equivalence(src_contents, dst_contents, mode=mode) return dst_contents @@ -948,7 +954,7 @@ def format_cell(src: str, *, fast: bool, mode: Mode) -> str: raise NothingChanged masked_dst = format_str(masked_src, mode=mode) if not fast: - check_src_and_dst_equivalent(masked_src, masked_dst, mode=mode) + check_stability_and_equivalence(masked_src, masked_dst, mode=mode) dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements) dst = put_trailing_semicolon_back( dst_without_trailing_semicolon, has_trailing_semicolon From 8bf86ba555bf30b2db3d36d1631df279ff1ef76b Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 12:48:00 +0100 Subject: [PATCH 74/81] make verbose and quiet keyword only --- src/black/__init__.py | 4 ++-- src/black/files.py | 2 +- tests/test_ipynb.py | 22 +++++++++++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index ecfcb420e99..6495f97f4ae 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -516,7 +516,7 @@ def get_sources( p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose, quiet + verbose=verbose, quiet=quiet ): continue @@ -739,7 +739,7 @@ async def schedule_formatting( @lru_cache() -def jupyter_dependencies_are_installed(verbose: bool, quiet: bool) -> bool: +def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: try: import IPython # noqa:F401 import tokenize_rt # noqa:F401 diff --git a/src/black/files.py b/src/black/files.py index 16ad4f02daa..5c5ed9edbe1 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -227,7 +227,7 @@ def gen_python_files( if child.suffix == ".ipynb": from black import jupyter_dependencies_are_installed - if not jupyter_dependencies_are_installed(): + if not jupyter_dependencies_are_installed(verbose=verbose, quiet=quiet): continue include_match = include.search(normalized_path) if include else True if include_match: diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 7e1bc167b15..fdd8c541dd2 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -365,7 +365,7 @@ def test_ipynb_diff_with_no_change() -> None: assert expected in result.output -def test_cache_isnt_written_if_no_jupyter_deps( +def test_cache_isnt_written_if_no_jupyter_deps_single( monkeypatch: MonkeyPatch, tmpdir: tmpdir ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. @@ -383,3 +383,23 @@ def test_cache_isnt_written_if_no_jupyter_deps( ) result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) assert "reformatted" in result.output + + +def test_cache_isnt_written_if_no_jupyter_deps_many( + monkeypatch: MonkeyPatch, tmpdir: tmpdir +) -> None: + # Check that the cache isn't written to if Jupyter dependencies aren't installed. + with open( + os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + ) as src, open(tmpdir / "notebook.ipynb", "w") as dst: + dst.write(src.read()) + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False + ) + result = runner.invoke(main, [str(tmpdir)]) + assert "No Python files are present to be formatted. Nothing to do" in result.output + monkeypatch.setattr( + "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True + ) + result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + assert "reformatted" in result.output From 46b802e06b8e3274ce9eb32d457bf905605b6feb Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 17:18:45 +0100 Subject: [PATCH 75/81] :art: --- tests/test_ipynb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index fdd8c541dd2..0a9b9da1711 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -369,9 +369,9 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( monkeypatch: MonkeyPatch, tmpdir: tmpdir ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. - with open( - os.path.join("tests", "data", "notebook_trailing_newline.ipynb") - ) as src, open(tmpdir / "notebook.ipynb", "w") as dst: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False @@ -389,9 +389,9 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( monkeypatch: MonkeyPatch, tmpdir: tmpdir ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. - with open( - os.path.join("tests", "data", "notebook_trailing_newline.ipynb") - ) as src, open(tmpdir / "notebook.ipynb", "w") as dst: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False From c0745419b089951837957fccca00edd0ea1313d7 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 21:00:33 +0100 Subject: [PATCH 76/81] run second many test on directory, not on file --- tests/test_ipynb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 0a9b9da1711..b88c4fb126e 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -401,5 +401,5 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True ) - result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) + result = runner.invoke(main, [str(tmpdir)]) assert "reformatted" in result.output From 72c6af152a1d59c4cfd27b0c3e67e200d2ab6c46 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 17 Jul 2021 22:19:51 +0100 Subject: [PATCH 77/81] test for warning message when running on directory --- tests/test_ipynb.py | 6 +++++- tests/test_no_ipynb.py | 25 ++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index b88c4fb126e..b812592480b 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,6 +1,6 @@ import pathlib from click.testing import CliRunner -from black import main +from black import main, jupyter_dependencies_are_installed from black import ( NothingChanged, format_cell, @@ -369,6 +369,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( monkeypatch: MonkeyPatch, tmpdir: tmpdir ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. + jupyter_dependencies_are_installed.cache_clear() nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") tmp_nb = tmpdir / "notebook.ipynb" with open(nb) as src, open(tmp_nb, "w") as dst: @@ -378,6 +379,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( ) result = runner.invoke(main, [str(tmpdir / "notebook.ipynb")]) assert "No Python files are present to be formatted. Nothing to do" in result.output + jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True ) @@ -389,6 +391,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( monkeypatch: MonkeyPatch, tmpdir: tmpdir ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. + jupyter_dependencies_are_installed.cache_clear() nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") tmp_nb = tmpdir / "notebook.ipynb" with open(nb) as src, open(tmp_nb, "w") as dst: @@ -398,6 +401,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( ) result = runner.invoke(main, [str(tmpdir)]) assert "No Python files are present to be formatted. Nothing to do" in result.output + jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True ) diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index bd48d32cf97..0ced78c3484 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,14 +1,18 @@ import pytest +import os from tests.util import THIS_DIR -from black import main +from black import main, jupyter_dependencies_are_installed from click.testing import CliRunner +from _pytest.tmpdir import tmpdir pytestmark = pytest.mark.no_jupyter +runner = CliRunner() -def test_ipynb_diff_with_no_change() -> None: - runner = CliRunner() + +def test_ipynb_diff_with_no_change_single() -> None: + jupyter_dependencies_are_installed.cache_clear() path = THIS_DIR / "data/notebook_trailing_newline.ipynb" result = runner.invoke(main, [str(path)]) expected_output = ( @@ -16,3 +20,18 @@ def test_ipynb_diff_with_no_change() -> None: "You can fix this by running ``pip install black[jupyter]``\n" ) assert expected_output in result.output + + +def test_ipynb_diff_with_no_change_many(tmpdir: tmpdir) -> None: + jupyter_dependencies_are_installed.cache_clear() + runner = CliRunner() + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + result = runner.invoke(main, [str(tmpdir)]) + expected_output = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``\n" + ) + assert expected_output in result.output From dad04e2315589f6d63533b087f82c6ab4a3f4402 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 18 Jul 2021 11:01:51 +0100 Subject: [PATCH 78/81] early return from non-python cell magics --- src/black/handle_ipynb_magics.py | 19 +++++++++++++++++++ tests/test_ipynb.py | 12 ++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 1d4861cc722..bb3c4d27630 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -10,6 +10,7 @@ from typing import Optional from typing_extensions import TypeGuard + TRANSFORMED_MAGICS = frozenset( ( "get_ipython().run_cell_magic", @@ -29,6 +30,22 @@ "ESCAPED_NL", ) ) +NON_PYTHON_CELL_MAGICS = frozenset( + ( + "%%bash", + "%%html", + "%%javascript", + "%%js", + "%%latex", + "%%markdown", + "%%perl", + "%%ruby", + "%%script", + "%%sh", + "%%svg", + "%%writefile", + ) +) @dataclasses.dataclass(frozen=True) @@ -194,6 +211,8 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: cell_magic_finder.visit(tree) if cell_magic_finder.cell_magic is None: return src, replacements + if cell_magic_finder.cell_magic.header.split()[0] in NON_PYTHON_CELL_MAGICS: + raise SyntaxError mask = get_token(src, cell_magic_finder.cell_magic.header) replacements.append(Replacement(mask=mask, src=cell_magic_finder.cell_magic.header)) return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index b812592480b..189af7e2095 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -95,6 +95,18 @@ def test_magic(src: str, expected: str) -> None: assert result == expected +@pytest.mark.parametrize( + "src", + ( + "%%bash\n2+2", + "%%html --isolated\n2+2", + ), +) +def test_non_python_magics(src: str) -> None: + with pytest.raises(NothingChanged): + format_cell(src, fast=True, mode=JUPYTER_MODE) + + def test_set_input() -> None: src = "a = b??" with pytest.raises(NothingChanged): From 3eab2ce71e5431c957407ead707271a5874d46bd Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 18 Jul 2021 11:18:01 +0100 Subject: [PATCH 79/81] move NothingChanged to report to avoid circular import --- src/black/__init__.py | 6 +----- src/black/handle_ipynb_magics.py | 25 +++++++++++-------------- src/black/report.py | 4 ++++ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 6495f97f4ae..a4b8351b92e 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -44,7 +44,7 @@ from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache from black.concurrency import cancel, shutdown, maybe_install_uvloop from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err -from black.report import Report, Changed +from black.report import Report, Changed, NothingChanged from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore from black.files import wrap_stream_for_windows @@ -71,10 +71,6 @@ NewLine = str -class NothingChanged(UserWarning): - """Raised when reformatted code is the same as source.""" - - class WriteBack(Enum): NO = 0 YES = 1 diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index bb3c4d27630..af67386f0da 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -9,6 +9,7 @@ from typing import Optional from typing_extensions import TypeGuard +from black.report import NothingChanged TRANSFORMED_MAGICS = frozenset( @@ -54,10 +55,6 @@ class Replacement: src: str -class UnsupportedMagic(UserWarning): - """Raise when Magic is not supported (e.g. `a = b??`)""" - - def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: """Remove trailing semicolon from Jupyter notebook cell. @@ -150,12 +147,10 @@ def mask_cell(src: str) -> Tuple[str, List[Replacement]]: transformed, cell_magic_replacements = replace_cell_magics(transformed) replacements += cell_magic_replacements transformed = transformer_manager.transform_cell(transformed) - try: - transformed, magic_replacements = replace_magics(transformed) - except UnsupportedMagic: - raise SyntaxError - if len(transformed.splitlines()) != len(src.splitlines()): # multi-line magic - raise SyntaxError + transformed, magic_replacements = replace_magics(transformed) + if len(transformed.splitlines()) != len(src.splitlines()): + # Multi-line magic, not supported. + raise NothingChanged replacements += magic_replacements return transformed, replacements @@ -212,7 +207,7 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]: if cell_magic_finder.cell_magic is None: return src, replacements if cell_magic_finder.cell_magic.header.split()[0] in NON_PYTHON_CELL_MAGICS: - raise SyntaxError + raise NothingChanged mask = get_token(src, cell_magic_finder.cell_magic.header) replacements.append(Replacement(mask=mask, src=cell_magic_finder.cell_magic.header)) return f"{mask}\n{cell_magic_finder.cell_magic.body}", replacements @@ -244,8 +239,10 @@ def replace_magics(src: str) -> Tuple[str, List[Replacement]]: if i in magic_finder.magics: offsets_and_magics = magic_finder.magics[i] if len(offsets_and_magics) != 1: # pragma: nocover - # defensive check - raise UnsupportedMagic + raise AssertionError( + f"Expecting one magic per line, got: {offsets_and_magics}\n" + "Please report a bug on https://github.com/psf/black/issues." + ) col_offset, magic = ( offsets_and_magics[0].col_offset, offsets_and_magics[0].magic, @@ -434,7 +431,7 @@ def visit_Expr(self, node: ast.Expr) -> None: elif node.value.func.attr == "getoutput": src = f"!!{args[0]}" else: - raise UnsupportedMagic + raise NothingChanged # unsupported magic. self.magics[node.value.lineno].append( OffsetAndMagic(node.value.col_offset, src) ) diff --git a/src/black/report.py b/src/black/report.py index 8fc5da2e167..7e1c8b4b87f 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -16,6 +16,10 @@ class Changed(Enum): YES = 2 +class NothingChanged(UserWarning): + """Raised when reformatted code is the same as source.""" + + @dataclass class Report: """Provides a reformatting counter. Can be rendered with `str(report)`.""" From eec268532559d7d1d3ebcf89014645b2a4c2f12f Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sun, 18 Jul 2021 11:38:33 +0100 Subject: [PATCH 80/81] remove circular import --- src/black/__init__.py | 19 +------------------ src/black/files.py | 10 +++++----- src/black/handle_ipynb_magics.py | 19 +++++++++++++++++++ tests/test_ipynb.py | 7 ++++--- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index a4b8351b92e..52b44b3738b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,5 +1,4 @@ import asyncio -from functools import lru_cache from json.decoder import JSONDecodeError import json from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor @@ -56,6 +55,7 @@ remove_trailing_semicolon, put_trailing_semicolon_back, TRANSFORMED_MAGICS, + jupyter_dependencies_are_installed, ) @@ -734,23 +734,6 @@ async def schedule_formatting( write_cache(cache, sources_to_cache, mode) -@lru_cache() -def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: - try: - import IPython # noqa:F401 - import tokenize_rt # noqa:F401 - except ModuleNotFoundError: - if verbose or not quiet: - msg = ( - "Skipping .ipynb files as Jupyter dependencies are not installed.\n" - "You can fix this by running ``pip install black[jupyter]``" - ) - out(msg) - return False - else: - return True - - def format_file_in_place( src: Path, fast: bool, diff --git a/src/black/files.py b/src/black/files.py index 5c5ed9edbe1..5e1ce03d0e6 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -22,6 +22,7 @@ from black.output import err from black.report import Report +from black.handle_ipynb_magics import jupyter_dependencies_are_installed if TYPE_CHECKING: import colorama # noqa: F401 @@ -224,11 +225,10 @@ def gen_python_files( ) elif child.is_file(): - if child.suffix == ".ipynb": - from black import jupyter_dependencies_are_installed - - if not jupyter_dependencies_are_installed(verbose=verbose, quiet=quiet): - continue + if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + verbose=verbose, quiet=quiet + ): + continue include_match = include.search(normalized_path) if include else True if include_match: yield child diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index af67386f0da..ad93c444efc 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,4 +1,5 @@ """Functions to process IPython magics with.""" +from functools import lru_cache import dataclasses import ast from typing import Dict @@ -10,6 +11,7 @@ from typing import Optional from typing_extensions import TypeGuard from black.report import NothingChanged +from black.output import out TRANSFORMED_MAGICS = frozenset( @@ -55,6 +57,23 @@ class Replacement: src: str +@lru_cache() +def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: + try: + import IPython # noqa:F401 + import tokenize_rt # noqa:F401 + except ModuleNotFoundError: + if verbose or not quiet: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + "You can fix this by running ``pip install black[jupyter]``" + ) + out(msg) + return False + else: + return True + + def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: """Remove trailing semicolon from Jupyter notebook cell. diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 189af7e2095..b9387be343a 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,7 +1,8 @@ import pathlib from click.testing import CliRunner -from black import main, jupyter_dependencies_are_installed +from black.handle_ipynb_magics import jupyter_dependencies_are_installed from black import ( + main, NothingChanged, format_cell, format_file_contents, @@ -409,13 +410,13 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( with open(nb) as src, open(tmp_nb, "w") as dst: dst.write(src.read()) monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False + "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False ) result = runner.invoke(main, [str(tmpdir)]) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True + "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: True ) result = runner.invoke(main, [str(tmpdir)]) assert "reformatted" in result.output From 67d38da8e6c9cd99c7deda059f1efc6f8d3ea049 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Fri, 30 Jul 2021 22:18:55 +0100 Subject: [PATCH 81/81] reinstate --ipynb flag --- .pre-commit-hooks.yaml | 11 +++++++++++ src/black/__init__.py | 15 ++++++++++++++- tests/test_ipynb.py | 41 +++++++++++++++++++++++++++++++++++++---- tests/test_no_ipynb.py | 2 +- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index de2eb674e0d..81848d7dcf7 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -7,3 +7,14 @@ minimum_pre_commit_version: 2.9.2 require_serial: true types_or: [python, pyi] +- id: black-jupyter + name: black-jupyter + description: + "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" + entry: black + language: python + language_version: python3 + minimum_pre_commit_version: 2.9.2 + require_serial: true + types_or: [python, pyi, jupyter] + additional_dependencies: [".[jupyter]"] diff --git a/src/black/__init__.py b/src/black/__init__.py index 52b44b3738b..29fb244f8b7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -203,6 +203,14 @@ def validate_regex( " when piping source on standard input)." ), ) +@click.option( + "--ipynb", + is_flag=True, + help=( + "Format all input files like Jupyter Notebooks regardless of file extension " + "(useful when piping source on standard input)." + ), +) @click.option( "-S", "--skip-string-normalization", @@ -362,6 +370,7 @@ def main( color: bool, fast: bool, pyi: bool, + ipynb: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, experimental_string_processing: bool, @@ -387,6 +396,9 @@ def main( f" the running version `{__version__}`!" ) ctx.exit(1) + if ipynb and pyi: + err("Cannot pass both `pyi` and `ipynb` flags!") + ctx.exit(1) write_back = WriteBack.from_configuration(check=check, diff=diff, color=color) if target_version: @@ -398,6 +410,7 @@ def main( target_versions=versions, line_length=line_length, is_pyi=pyi, + is_ipynb=ipynb, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, experimental_string_processing=experimental_string_processing, @@ -769,7 +782,7 @@ def format_file_in_place( now = datetime.utcnow() src_name = f"{src}\t{then} +0000" dst_name = f"{src}\t{now} +0000" - if src.suffix == ".ipynb": + if mode.is_ipynb: diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) else: diff_contents = diff(src_contents, dst_contents, src_name, dst_name) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index b9387be343a..038155e9270 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -12,7 +12,7 @@ import pytest from black import Mode from _pytest.monkeypatch import MonkeyPatch -from _pytest.tmpdir import tmpdir +from py.path import local pytestmark = pytest.mark.jupyter pytest.importorskip("IPython", reason="IPython is an optional dependency") @@ -379,7 +379,7 @@ def test_ipynb_diff_with_no_change() -> None: def test_cache_isnt_written_if_no_jupyter_deps_single( - monkeypatch: MonkeyPatch, tmpdir: tmpdir + monkeypatch: MonkeyPatch, tmpdir: local ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. jupyter_dependencies_are_installed.cache_clear() @@ -400,8 +400,8 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( assert "reformatted" in result.output -def test_cache_isnt_written_if_no_jupyter_deps_many( - monkeypatch: MonkeyPatch, tmpdir: tmpdir +def test_cache_isnt_written_if_no_jupyter_deps_dir( + monkeypatch: MonkeyPatch, tmpdir: local ) -> None: # Check that the cache isn't written to if Jupyter dependencies aren't installed. jupyter_dependencies_are_installed.cache_clear() @@ -420,3 +420,36 @@ def test_cache_isnt_written_if_no_jupyter_deps_many( ) result = runner.invoke(main, [str(tmpdir)]) assert "reformatted" in result.output + + +def test_ipynb_flag(tmpdir: local) -> None: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + tmp_nb = tmpdir / "notebook.a_file_extension_which_is_definitely_not_ipynb" + with open(nb) as src, open(tmp_nb, "w") as dst: + dst.write(src.read()) + result = runner.invoke( + main, + [ + str(tmp_nb), + "--diff", + "--ipynb", + ], + ) + expected = "@@ -1,3 +1,3 @@\n %%time\n \n-print('foo')\n" '+print("foo")\n' + assert expected in result.output + + +def test_ipynb_and_pyi_flags() -> None: + nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb") + result = runner.invoke( + main, + [ + nb, + "--pyi", + "--ipynb", + "--diff", + ], + ) + assert isinstance(result.exception, SystemExit) + expected = "Cannot pass both `pyi` and `ipynb` flags!\n" + assert result.output == expected diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index 0ced78c3484..bcda2d5369f 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -22,7 +22,7 @@ def test_ipynb_diff_with_no_change_single() -> None: assert expected_output in result.output -def test_ipynb_diff_with_no_change_many(tmpdir: tmpdir) -> None: +def test_ipynb_diff_with_no_change_dir(tmpdir: tmpdir) -> None: jupyter_dependencies_are_installed.cache_clear() runner = CliRunner() nb = os.path.join("tests", "data", "notebook_trailing_newline.ipynb")