From c2ddf0adaa23cd4f8ac154af97f1763cea44d604 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Thu, 22 Oct 2020 17:08:42 -0300 Subject: [PATCH 01/10] Provide a stdin-filename to allow stdin to respect exclude/force-exclude rules This will allow automatic tools to enforce the project's exclude/force-exclude rules even if they pass the file through stdin to update its buffer. This is a similar solution to --stdin-display-name in flake8. --- src/black/__init__.py | 31 +++++++++++++++++++++++++++---- tests/test_black.py | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 24e9d4edaaa..7fab0656d19 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -457,6 +457,15 @@ def target_version_option_callback( "excluded even when they are passed explicitly as arguments." ), ) +@click.option( + "--stdin-filename", + type=str, + help=( + "The name of the file when passing it through stdin. Useful to make " + "sure black will respect --exclude/--force-exclude options on some " + "editors that rely on using stdin." + ), +) @click.option( "-q", "--quiet", @@ -516,6 +525,7 @@ def main( include: str, exclude: str, force_exclude: Optional[str], + stdin_filename: Optional[str], src: Tuple[str, ...], config: Optional[str], ) -> None: @@ -548,6 +558,7 @@ def main( exclude=exclude, force_exclude=force_exclude, report=report, + stdin_filename=stdin_filename, ) path_empty( @@ -587,6 +598,7 @@ def get_sources( exclude: str, force_exclude: Optional[str], report: "Report", + stdin_filename: Optional[str], ) -> Set[Path]: """Compute the set of files to be formatted.""" try: @@ -613,7 +625,13 @@ def get_sources( gitignore = get_gitignore(root) for s in src: - p = Path(s) + if s == "-" and stdin_filename: + p = Path(stdin_filename) + is_stdin = True + else: + p = Path(s) + is_stdin = False + if p.is_dir(): sources.update( gen_python_files( @@ -626,8 +644,6 @@ def get_sources( gitignore, ) ) - elif s == "-": - sources.add(p) elif p.is_file(): normalized_path = normalize_path_maybe_ignore(p, root, report) if normalized_path is None: @@ -643,6 +659,11 @@ def get_sources( report.path_ignored(p, "matches the --force-exclude regular expression") continue + if is_stdin: + p = Path("__BLACK_STDIN_FILENAME__" / p) + + sources.add(p) + elif s == "-": sources.add(p) else: err(f"invalid path: {s}") @@ -670,7 +691,9 @@ def reformat_one( """ try: changed = Changed.NO - if not src.is_file() and str(src) == "-": + if not src.is_file() and ( + str(src) == "-" or str(src).startswith("__BLACK_STDIN_FILENAME__") + ): if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: diff --git a/tests/test_black.py b/tests/test_black.py index 7927b368b5d..ed108671836 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1730,6 +1730,7 @@ def test_exclude_for_issue_1572(self) -> None: exclude=exclude, force_exclude=None, report=report, + stdin_filename=None, ) ) self.assertEqual(sorted(expected), sorted(sources)) From f9b361a6077fec8ed49a0283f1f4cf13383a6d80 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Fri, 23 Oct 2020 10:08:00 -0300 Subject: [PATCH 02/10] Update src/black/__init__.py Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 7fab0656d19..1b7176b38cf 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -644,7 +644,7 @@ def get_sources( gitignore, ) ) - elif p.is_file(): + elif p.is_file() or is_stdin: normalized_path = normalize_path_maybe_ignore(p, root, report) if normalized_path is None: continue From d728f2c811dc5f0e4342568a2d4ba0349a3647ba Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Fri, 23 Oct 2020 10:13:03 -0300 Subject: [PATCH 03/10] --stdin-filename should only respect --exclude-filename --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 1b7176b38cf..0b52f2f835b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -462,7 +462,7 @@ def target_version_option_callback( type=str, help=( "The name of the file when passing it through stdin. Useful to make " - "sure black will respect --exclude/--force-exclude options on some " + "sure black will respect --force-exclude option on some " "editors that rely on using stdin." ), ) From aae2c4332609332efb408744ee89c54b4b5f034d Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Fri, 23 Oct 2020 10:13:50 -0300 Subject: [PATCH 04/10] Update README with the new --stdin-filename option --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 47472810114..b0fe97f8a9f 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,11 @@ Options: matching this regex will be excluded even when they are passed explicitly as arguments. + --stdin-filename TEXT The name of the file when passing it through + stdin. Useful to make sure black will respect + --force-exclude option on some editors that + rely on using stdin. + -q, --quiet Don't emit non-error messages to stderr. Errors are still emitted; silence those with 2>/dev/null. From 7addb032b63015b58d294990d296622c7b630fb6 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Fri, 23 Oct 2020 11:20:39 -0300 Subject: [PATCH 05/10] Write some tests for the new stdin-filename functionality --- src/black/__init__.py | 17 ++++-- tests/test_black.py | 134 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0b52f2f835b..63761130902 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -68,6 +68,7 @@ DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist)/" # noqa: B950 DEFAULT_INCLUDES = r"\.pyi?$" CACHE_DIR = Path(user_cache_dir("black", version=__version__)) +STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. @@ -660,7 +661,7 @@ def get_sources( continue if is_stdin: - p = Path("__BLACK_STDIN_FILENAME__" / p) + p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") sources.add(p) elif s == "-": @@ -691,9 +692,17 @@ def reformat_one( """ try: changed = Changed.NO - if not src.is_file() and ( - str(src) == "-" or str(src).startswith("__BLACK_STDIN_FILENAME__") - ): + is_stdin = False + + if str(src) == "-": + is_stdin = True + elif str(src).startswith(STDIN_PLACEHOLDER): + is_stdin = True + # Use the original name again in case we want to print something + # to the user + src = Path(str(src)[len(STDIN_PLACEHOLDER) :]) + + if not src.is_file() and is_stdin: if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: diff --git a/tests/test_black.py b/tests/test_black.py index ed108671836..21c9e77261a 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1735,6 +1735,140 @@ def test_exclude_for_issue_1572(self) -> None: ) self.assertEqual(sorted(expected), sorted(sources)) + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) + def test_get_sources_with_stdin(self) -> None: + include = "" + exclude = r"/exclude/|a\.py" + src = "-" + report = black.Report() + expected = [Path("-")] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude=exclude, + force_exclude=None, + report=report, + stdin_filename=None, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) + def test_get_sources_with_stdin_filename(self) -> None: + include = "" + exclude = r"/exclude/|a\.py" + src = "-" + report = black.Report() + stdin_filename = str(THIS_DIR / "data/collections.py") + expected = [Path(f"__BLACK_STDIN_FILENAME__{stdin_filename}")] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude=exclude, + force_exclude=None, + report=report, + stdin_filename=stdin_filename, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) + def test_get_sources_with_stdin_filename_and_exclude(self) -> None: + # Exclude shouldn't exclude stdin_filename since it is mimicing the + # file being passed directly. This is the same as + # test_exclude_for_issue_1572 + path = THIS_DIR / "data" / "include_exclude_tests" + include = "" + exclude = r"/exclude/|a\.py" + src = "-" + report = black.Report() + stdin_filename = str(path / "b/exclude/a.py") + expected = [Path(f"__BLACK_STDIN_FILENAME__{stdin_filename}")] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude=exclude, + force_exclude=None, + report=report, + stdin_filename=stdin_filename, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) + def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: + # Force exclude should exclude the file when passing it through + # stdin_filename + path = THIS_DIR / "data" / "include_exclude_tests" + include = "" + force_exclude = r"/exclude/|a\.py" + src = "-" + report = black.Report() + stdin_filename = str(path / "b/exclude/a.py") + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude="", + force_exclude=force_exclude, + report=report, + stdin_filename=stdin_filename, + ) + ) + self.assertEqual([], sorted(sources)) + + def test_reformat_one_with_stdin(self) -> None: + with patch( + "black.format_stdin_to_stdout", + return_value=lambda *args, **kwargs: black.Changed.YES, + ) as fsts: + report = MagicMock() + path = Path("-") + black.reformat_one( + path, + fast=True, + write_back=black.WriteBack.YES, + mode=DEFAULT_MODE, + report=report, + ) + fsts.assert_called_once() + report.done.assert_called_with(path, black.Changed.YES) + + def test_reformat_one_with_stdin_filename(self) -> None: + with patch( + "black.format_stdin_to_stdout", + return_value=lambda *args, **kwargs: black.Changed.YES, + ) as fsts: + report = MagicMock() + p = "foo.py" + path = Path(f"__BLACK_STDIN_FILENAME__{p}") + expected = Path(p) + black.reformat_one( + path, + fast=True, + write_back=black.WriteBack.YES, + mode=DEFAULT_MODE, + report=report, + ) + fsts.assert_called_once() + # __BLACK_STDIN_FILENAME__ should have been striped + report.done.assert_called_with(expected, black.Changed.YES) + def test_gitignore_exclude(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" include = re.compile(r"\.pyi?$") From 72477b2560e1dfe1cf0f9bf0e42eaed72281be52 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Fri, 23 Oct 2020 11:28:39 -0300 Subject: [PATCH 06/10] Apply suggestions from code review Co-authored-by: Hugo van Kemenade --- README.md | 2 +- src/black/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0fe97f8a9f..0615ba6aa3a 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Options: when they are passed explicitly as arguments. --stdin-filename TEXT The name of the file when passing it through - stdin. Useful to make sure black will respect + stdin. Useful to make sure Black will respect --force-exclude option on some editors that rely on using stdin. diff --git a/src/black/__init__.py b/src/black/__init__.py index 63761130902..2d4c310aaeb 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -463,7 +463,7 @@ def target_version_option_callback( type=str, help=( "The name of the file when passing it through stdin. Useful to make " - "sure black will respect --force-exclude option on some " + "sure Black will respect --force-exclude option on some " "editors that rely on using stdin." ), ) From 00e80285c5cd93501c619b15cbc812cce43106a9 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Sun, 25 Oct 2020 13:36:42 -0300 Subject: [PATCH 07/10] Force stdin output when we asked for stdin even if the file exists --- src/black/__init__.py | 5 +++-- tests/test_black.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 2d4c310aaeb..8f557259e67 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -692,7 +692,6 @@ def reformat_one( """ try: changed = Changed.NO - is_stdin = False if str(src) == "-": is_stdin = True @@ -701,8 +700,10 @@ def reformat_one( # Use the original name again in case we want to print something # to the user src = Path(str(src)[len(STDIN_PLACEHOLDER) :]) + else: + is_stdin = False - if not src.is_file() and is_stdin: + if is_stdin: if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: diff --git a/tests/test_black.py b/tests/test_black.py index 21c9e77261a..2eda10c033f 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1869,6 +1869,30 @@ def test_reformat_one_with_stdin_filename(self) -> None: # __BLACK_STDIN_FILENAME__ should have been striped report.done.assert_called_with(expected, black.Changed.YES) + def test_reformat_one_with_stdin_and_existing_path(self) -> None: + with patch( + "black.format_stdin_to_stdout", + return_value=lambda *args, **kwargs: black.Changed.YES, + ) as fsts: + report = MagicMock() + # Even with an existing file, since we are forcing stdin, black + # should output to stdout and not modify the file inplace + p = Path(str(THIS_DIR / "data/collections.py")) + # Make sure is_file actually returns True + self.assertTrue(p.is_file()) + path = Path(f"__BLACK_STDIN_FILENAME__{p}") + expected = Path(p) + black.reformat_one( + path, + fast=True, + write_back=black.WriteBack.YES, + mode=DEFAULT_MODE, + report=report, + ) + fsts.assert_called_once() + # __BLACK_STDIN_FILENAME__ should have been striped + report.done.assert_called_with(expected, black.Changed.YES) + def test_gitignore_exclude(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" include = re.compile(r"\.pyi?$") From 2ac3baebb2087efa70960a91c868f06586e18f62 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Sun, 25 Oct 2020 13:44:04 -0300 Subject: [PATCH 08/10] Add an entry in the changelog regarding --stdin-filename --- docs/change_log.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/change_log.md b/docs/change_log.md index 1ee35a4d8f9..e6afefbf686 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -23,6 +23,9 @@ - Added support for PEP 614 relaxed decorator syntax on python 3.9 (#1711) +- Added `--stdin-filename` argument to allow stdin to respect `--force-exclude` rules. + Works very alike to flake8's `--stdin-display-name` (#1780) + ### 20.8b1 #### _Packaging_ From 8d13a9c95bb0fd5160d73fd3b5d3d5326ed46215 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Wed, 4 Nov 2020 11:27:25 -0300 Subject: [PATCH 09/10] Reduce disk reads if possible Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- src/black/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 8f557259e67..7594ccea82b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -633,7 +633,7 @@ def get_sources( p = Path(s) is_stdin = False - if p.is_dir(): + if not is_stdin or p.is_dir(): sources.update( gen_python_files( p.iterdir(), @@ -645,7 +645,7 @@ def get_sources( gitignore, ) ) - elif p.is_file() or is_stdin: + elif is_stdin or p.is_file(): normalized_path = normalize_path_maybe_ignore(p, root, report) if normalized_path is None: continue From 7320a10ec942fd8256192f0714e6db9f8d304aa6 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Ribeiro Date: Wed, 4 Nov 2020 20:14:19 -0300 Subject: [PATCH 10/10] Check for is_stdin and p.is_file before checking for p.is_dir() --- src/black/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 7594ccea82b..0850b115b37 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -633,19 +633,7 @@ def get_sources( p = Path(s) is_stdin = False - if not is_stdin or p.is_dir(): - sources.update( - gen_python_files( - p.iterdir(), - root, - include_regex, - exclude_regex, - force_exclude_regex, - report, - gitignore, - ) - ) - elif is_stdin or p.is_file(): + if is_stdin or p.is_file(): normalized_path = normalize_path_maybe_ignore(p, root, report) if normalized_path is None: continue @@ -664,6 +652,18 @@ def get_sources( p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") sources.add(p) + elif p.is_dir(): + sources.update( + gen_python_files( + p.iterdir(), + root, + include_regex, + exclude_regex, + force_exclude_regex, + report, + gitignore, + ) + ) elif s == "-": sources.add(p) else: