From eacbf702e1f2c61f89fb276444666f091697e7f8 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 6 Sep 2019 15:31:05 -0700 Subject: [PATCH] Make black able to compile and run with mypyc The changes made fall into a couple categories: * Fixing actual type mistakes that slip through the cracks * Working around a couple mypy bugs (the most annoying of which being that we need to add type annotations in a number of places where variables are initialized to None Co-authored-by: Sanjit Kalapatapu Co-authored-by: Michael J. Sullivan --- .flake8 | 4 ++++ Pipfile | 2 ++ Pipfile.lock | 28 +++++++++++++++++++++++----- black.py | 41 ++++++++++++++++++++++++++++------------- setup.py | 29 +++++++++++++++++++++++++++++ tests/test_black.py | 1 + 6 files changed, 87 insertions(+), 18 deletions(-) diff --git a/.flake8 b/.flake8 index c321e71c965..498b2cb30f7 100644 --- a/.flake8 +++ b/.flake8 @@ -3,3 +3,7 @@ ignore = E203, E266, E501, W503 max-line-length = 80 max-complexity = 18 select = B,C,E,F,W,T4,B9 +# We need to configure the mypy.ini because the flake8-mypy's default +# options don't properly override it, so if we don't specify it we get +# half of the config from mypy.ini and half from flake8-mypy. +mypy_config = mypy.ini diff --git a/Pipfile b/Pipfile index 3784810e5a0..a9bb7cad6cf 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,8 @@ toml = ">=0.9.4" black = {path = ".",extras = ["d"],editable = true} aiohttp-cors = "*" typed-ast = "==1.4.0" +typing_extensions = ">=3.7.4" +mypy_extensions = ">=0.4.3" regex = ">=2019.8" pathspec = ">=0.6" diff --git a/Pipfile.lock b/Pipfile.lock index abcea4855b2..4f92c0ce1cb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0ae3ab3331dc4448fc4def999af38d91c13a566a482a6935c4a880f6bd9a6614" + "sha256": "f680cb02a00b1b4a814bdb2d4d3eeaf9e1225bb96a2352e5807d8e08fc88e885" }, "pipfile-spec": 6, "requires": {}, @@ -126,6 +126,14 @@ ], "version": "==4.5.2" }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "index": "pypi", + "version": "==0.4.3" + }, "pathspec": { "hashes": [ "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" @@ -186,6 +194,15 @@ "index": "pypi", "version": "==1.4.0" }, + "typing-extensions": { + "hashes": [ + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" + ], + "index": "pypi", + "version": "==3.7.4.1" + }, "yarl": { "hashes": [ "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", @@ -689,11 +706,12 @@ }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], - "version": "==3.7.4" + "index": "pypi", + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ diff --git a/black.py b/black.py index 4e7d6165f0d..0f1de5b7e6e 100644 --- a/black.py +++ b/black.py @@ -37,6 +37,7 @@ Union, cast, ) +from mypy_extensions import mypyc_attr from appdirs import user_cache_dir from attr import dataclass, evolve, Factory @@ -247,6 +248,17 @@ def read_pyproject_toml( return value +def target_version_option_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] +) -> List[TargetVersion]: + """Compute the target versions from a --target-version flag. + + This is its own function because mypy couldn't infer the type correctly + when it was a lambda, causing mypyc trouble. + """ + return [TargetVersion[val.upper()] for val in v] + + @click.command(context_settings=dict(help_option_names=["-h", "--help"])) @click.option("-c", "--code", type=str, help="Format the code passed in as a string.") @click.option( @@ -261,7 +273,7 @@ def read_pyproject_toml( "-t", "--target-version", type=click.Choice([v.name.lower() for v in TargetVersion]), - callback=lambda c, p, v: [TargetVersion[val.upper()] for val in v], + callback=target_version_option_callback, multiple=True, help=( "Python versions that should be supported by Black's output. [default: " @@ -388,7 +400,7 @@ def main( verbose: bool, include: str, exclude: str, - src: Tuple[str], + src: Tuple[str, ...], config: Optional[str], ) -> None: """The uncompromising code formatter.""" @@ -470,7 +482,9 @@ def main( ctx.exit(report.return_code) -def path_empty(src: Tuple[str], quiet: bool, verbose: bool, ctx: click.Context) -> None: +def path_empty( + src: Tuple[str, ...], quiet: bool, verbose: bool, ctx: click.Context +) -> None: """ Exit if there is no `src` provided for formatting """ @@ -639,10 +653,10 @@ def format_file_in_place( except NothingChanged: return False - if write_back == write_back.YES: + if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: f.write(dst_contents) - elif write_back == write_back.DIFF: + elif write_back == WriteBack.DIFF: now = datetime.utcnow() src_name = f"{src}\t{then} +0000" dst_name = f"{src}\t{now} +0000" @@ -1728,13 +1742,13 @@ def visit_default(self, node: LN) -> Iterator[Line]: self.current_line.append(node) yield from super().visit_default(node) - def visit_INDENT(self, node: Node) -> Iterator[Line]: + def visit_INDENT(self, node: Leaf) -> Iterator[Line]: """Increase indentation level, maybe yield a line.""" # In blib2to3 INDENT never holds comments. yield from self.line(+1) yield from self.visit_default(node) - def visit_DEDENT(self, node: Node) -> Iterator[Line]: + def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: """Decrease indentation level, maybe yield a line.""" # The current line might still wait for trailing comments. At DEDENT time # there won't be any (they would be prefixes on the preceding NEWLINE). @@ -2462,7 +2476,7 @@ def left_hand_split(line: Line, features: Collection[Feature] = ()) -> Iterator[ body_leaves: List[Leaf] = [] head_leaves: List[Leaf] = [] current_leaves = head_leaves - matching_bracket = None + matching_bracket: Optional[Leaf] = None for leaf in line.leaves: if ( current_leaves is body_leaves @@ -2505,8 +2519,8 @@ def right_hand_split( body_leaves: List[Leaf] = [] head_leaves: List[Leaf] = [] current_leaves = tail_leaves - opening_bracket = None - closing_bracket = None + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None for leaf in reversed(line.leaves): if current_leaves is body_leaves: if leaf is opening_bracket: @@ -3027,7 +3041,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: # That happens when one of the `ignored_nodes` ended with a NEWLINE # leaf (possibly followed by a DEDENT). hidden_value = hidden_value[:-1] - first_idx = None + first_idx: Optional[int] = None for ignored in ignored_nodes: index = ignored.remove() if first_idx is None: @@ -3398,8 +3412,8 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf yield omit length = 4 * line.depth - opening_bracket = None - closing_bracket = None + opening_bracket: Optional[Leaf] = None + closing_bracket: Optional[Leaf] = None inner_brackets: Set[LeafID] = set() for index, leaf, leaf_length in enumerate_with_length(line, reversed=True): length += leaf_length @@ -3796,6 +3810,7 @@ def assert_stable(src: str, dst: str, mode: FileMode) -> None: ) from None +@mypyc_attr(patchable=True) def dump_to_file(*output: str) -> str: """Dump `output` to a temporary file. Return path to the file.""" with tempfile.NamedTemporaryFile( diff --git a/setup.py b/setup.py index 614a8d6051e..3196d0eb924 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # Copyright (C) 2018 Łukasz Langa from setuptools import setup import sys +import os assert sys.version_info >= (3, 6, 0), "black requires Python 3.6+" from pathlib import Path # noqa E402 @@ -15,6 +16,33 @@ def get_long_description() -> str: return ld_file.read() +USE_MYPYC = False +# To compile with mypyc, a mypyc checkout must be present on the PYTHONPATH +if len(sys.argv) > 1 and sys.argv[1] == "--use-mypyc": + sys.argv.pop(1) + USE_MYPYC = True +if os.getenv("BLACK_USE_MYPYC", None) == "1": + USE_MYPYC = True + +if USE_MYPYC: + mypyc_targets = [ + "black.py", + "blib2to3/pytree.py", + "blib2to3/pygram.py", + "blib2to3/pgen2/parse.py", + "blib2to3/pgen2/grammar.py", + "blib2to3/pgen2/token.py", + "blib2to3/pgen2/driver.py", + "blib2to3/pgen2/pgen.py", + ] + + from mypyc.build import mypycify, MypycifyBuildExt + + opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") + ext_modules = mypycify(mypyc_targets, opt_level=opt_level) +else: + ext_modules = [] + setup( name="black", use_scm_version={ @@ -30,6 +58,7 @@ def get_long_description() -> str: url="https://github.com/psf/black", license="MIT", py_modules=["black", "blackd", "_black_version"], + ext_modules=ext_modules, packages=["blib2to3", "blib2to3.pgen2"], package_data={"blib2to3": ["*.txt"]}, python_requires=">=3.6", diff --git a/tests/test_black.py b/tests/test_black.py index 93f853bae4f..40bde361eeb 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1540,6 +1540,7 @@ def test_symlink_out_of_root_directory(self) -> None: # outside of the `root` directory. path.iterdir.return_value = [child] child.resolve.return_value = Path("/a/b/c") + child.as_posix.return_value = "/a/b/c" child.is_symlink.return_value = True try: list(