From b0100b5d91c2f5db544a60f34aafab120f0aa458 Mon Sep 17 00:00:00 2001 From: temeddix <66480156+temeddix@users.noreply.github.com> Date: Fri, 14 May 2021 03:35:02 +0900 Subject: [PATCH 01/11] Solved Problem with non-alphabetical .gitignore files When .gitignore file in the user's project directory contained non-alphabetical characters(Japanese, Korean, Chinese, etc), Nothing works and printed this weird message in the console('cp949' is the encoding for Korean characters in this case). It even blocks VSCode's formatting from working. This commit solves the problem. Traceback (most recent call last): File "c:\users\username\anaconda3\envs\project-name\lib\runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "c:\users\username\anaconda3\envs\project-name\lib\runpy.py", line 85, in _run_code exec(code, run_globals) File "C:\Users\username\anaconda3\envs\project-name\Scripts\black.exe\__main__.py", line 7, in File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\black\__init__.py", line 1056, in patched_main main() File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\click\core.py", line 829, in __call__ return self.main(*args, **kwargs) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\click\core.py", line 782, in main rv = self.invoke(ctx) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\click\core.py", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\click\core.py", line 610, in invoke return callback(*args, **kwargs) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\click\decorators.py", line 21, in new_func return f(get_current_context(), *args, **kwargs) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\black\__init__.py", line 394, in main stdin_filename=stdin_filename, File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\black\__init__.py", line 445, in get_sources gitignore = get_gitignore(root) File "c:\users\username\anaconda3\envs\project-name\lib\site-packages\black\files.py", line 122, in get_gitignore lines = gf.readlines() UnicodeDecodeError: 'cp949' codec can't decode byte 0xb0 in position 13: illegal multibyte sequence --- src/black/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/files.py b/src/black/files.py index 1be560643a1..7c32f67ee21 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -118,7 +118,7 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with gitignore.open() as gf: + with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines) From 6c3a7ea42b5b1e441cc0026c8205d1cee68c1bba Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 04:38:51 +0900 Subject: [PATCH 02/11] Made .gitignore File Reader Detect Its Encoding --- src/black/files.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/black/files.py b/src/black/files.py index 7c32f67ee21..03030d4b0af 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -3,6 +3,7 @@ import os from pathlib import Path import sys +import tokenize from typing import ( Any, Dict, @@ -118,8 +119,13 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with gitignore.open(encoding="utf-8") as gf: + with open(gitignore, "rb") as buf: + srcbuf = io.BytesIO(buf.read()) + encoding, _ = tokenize.detect_encoding(srcbuf.readline) + # Detect Encoding + with gitignore.open(encoding=encoding) as gf: lines = gf.readlines() + # Open .gitignore file with detected encoding return PathSpec.from_lines("gitwildmatch", lines) From fa3c5deebf58b55ac62bfee14c12067409b8f13b Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 14:43:53 +0900 Subject: [PATCH 03/11] Revert "Made .gitignore File Reader Detect Its Encoding" This reverts commit 6c3a7ea42b5b1e441cc0026c8205d1cee68c1bba. --- src/black/files.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/black/files.py b/src/black/files.py index 03030d4b0af..7c32f67ee21 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -3,7 +3,6 @@ import os from pathlib import Path import sys -import tokenize from typing import ( Any, Dict, @@ -119,13 +118,8 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with open(gitignore, "rb") as buf: - srcbuf = io.BytesIO(buf.read()) - encoding, _ = tokenize.detect_encoding(srcbuf.readline) - # Detect Encoding - with gitignore.open(encoding=encoding) as gf: + with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() - # Open .gitignore file with detected encoding return PathSpec.from_lines("gitwildmatch", lines) From 8c21ba8ae5c1cce6ac4e1647f8913c9275eb63eb Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 14:43:57 +0900 Subject: [PATCH 04/11] Revert "Solved Problem with non-alphabetical .gitignore files" This reverts commit b0100b5d91c2f5db544a60f34aafab120f0aa458. --- src/black/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/files.py b/src/black/files.py index 7c32f67ee21..1be560643a1 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -118,7 +118,7 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with gitignore.open(encoding="utf-8") as gf: + with gitignore.open() as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines) From 50dd80422938649ccc8c7f43aac752f9f6481779 Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 14:45:51 +0900 Subject: [PATCH 05/11] Made .gitignore Reader Open the File with Auto Encoding Detecting https://docs.python.org/3.8/library/tokenize.html#tokenize.open --- src/black/files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/black/files.py b/src/black/files.py index 1be560643a1..7767874d8fe 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -3,6 +3,7 @@ import os from pathlib import Path import sys +import tokenize from typing import ( Any, Dict, @@ -118,7 +119,7 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with gitignore.open() as gf: + with tokenize.open(gitignore) as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines) From b848bce2942afccbe894c9007580f2351503cf66 Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 14:58:37 +0900 Subject: [PATCH 06/11] Revert "Made .gitignore Reader Open the File with Auto Encoding Detecting" This reverts commit 50dd80422938649ccc8c7f43aac752f9f6481779. --- src/black/files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/black/files.py b/src/black/files.py index 7767874d8fe..1be560643a1 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -3,7 +3,6 @@ import os from pathlib import Path import sys -import tokenize from typing import ( Any, Dict, @@ -119,7 +118,7 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with tokenize.open(gitignore) as gf: + with gitignore.open() as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines) From 9cde20a0a7d0d0d75ae54a843b472e28f7dcbfc8 Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 14:59:35 +0900 Subject: [PATCH 07/11] Made .gitignore Reader Use UTF-8 --- src/black/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/files.py b/src/black/files.py index 1be560643a1..7c32f67ee21 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -118,7 +118,7 @@ def get_gitignore(root: Path) -> PathSpec: gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): - with gitignore.open() as gf: + with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines) From ae88e22903a7546743cedcfe122e7dbc5ab81bfe Mon Sep 17 00:00:00 2001 From: temeddix Date: Fri, 14 May 2021 23:43:16 +0900 Subject: [PATCH 08/11] Updated CHANGES.md for #2229 --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3639d8ebaf9..8790419df92 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ ### _Black_ +- Fixed bug that Black doesn't do anything when + .gitignore file includes non-alphabetical(non-ascii) characters (#2229) - Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) From 423f1cc9a8937f2e9618464dff7a6c5b08b27405 Mon Sep 17 00:00:00 2001 From: temeddix <66480156+temeddix@users.noreply.github.com> Date: Sat, 15 May 2021 01:05:32 +0900 Subject: [PATCH 09/11] Updated CHANGES.md for #2229 Co-authored-by: Jelle Zijlstra --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8790419df92..7a81afae8a8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,8 +4,7 @@ ### _Black_ -- Fixed bug that Black doesn't do anything when - .gitignore file includes non-alphabetical(non-ascii) characters (#2229) +- Fix handling of .gitignore files containing non-ASCII characters (#2229) - Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) From 8c92b3d803cad42d0b0715a263ed721019e10155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sun, 16 May 2021 18:05:20 +0200 Subject: [PATCH 10/11] Update CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7a81afae8a8..11949e8116d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ ### _Black_ -- Fix handling of .gitignore files containing non-ASCII characters (#2229) +- Fix handling of .gitignore files containing non-ASCII characters on Windows (#2229) - Fix typos discovered by codespell (#2228) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227) From f63781213175cff7f0a80648b27d5f9fbe3f06be Mon Sep 17 00:00:00 2001 From: temeddix <66480156+temeddix@users.noreply.github.com> Date: Mon, 17 May 2021 04:50:50 +0900 Subject: [PATCH 11/11] Update CHANGES.md Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- CHANGES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fbef655bc0b..0bdccb23c3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,6 @@ ### _Black_ - Fix handling of .gitignore files containing non-ASCII characters on Windows (#2229) -- Fix typos discovered by codespell (#2228) - Respect `.gitignore` files in all levels, not only `root/.gitignore` file (apply `.gitignore` rules like `git` does) (#2225) - Restored compatibility with Click 8.0 on Python 3.6 when LANG=C used (#2227)