From b84b8a2f681ded10cadf9a118fb883ff0a6f8d27 Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sat, 3 Dec 2022 11:06:59 +0900 Subject: [PATCH 1/9] avoid lib2to3 --- autopep8.py | 136 ++++++------------------------------------ test/test_autopep8.py | 31 ---------- 2 files changed, 19 insertions(+), 148 deletions(-) diff --git a/autopep8.py b/autopep8.py index b6bc7e28..6d27445d 100755 --- a/autopep8.py +++ b/autopep8.py @@ -106,6 +106,9 @@ class documentation for more information. DOCSTRING_START_REGEX = re.compile(r'^u?r?(?P["\']{3})') ENABLE_REGEX = re.compile(r'# *(fmt|autopep8): *on') DISABLE_REGEX = re.compile(r'# *(fmt|autopep8): *off') +ENCODING_MAGIC_COMMENT = re.compile( + r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)' +) EXIT_CODE_OK = 0 EXIT_CODE_ERROR = 1 @@ -129,25 +132,6 @@ class documentation for more information. # to be enabled, disable both of them CONFLICTING_CODES = ('W503', 'W504') -# W602 is handled separately due to the need to avoid "with_traceback". -CODE_TO_2TO3 = { - 'E231': ['ws_comma'], - 'E721': ['idioms'], - 'W690': ['apply', - 'except', - 'exitfunc', - 'numliterals', - 'operator', - 'paren', - 'reduce', - 'renames', - 'standarderror', - 'sys_exc', - 'throw', - 'tuple_params', - 'xreadlines']} - - if sys.platform == 'win32': # pragma: no cover DEFAULT_CONFIG = os.path.expanduser(r'~\.pycodestyle') else: @@ -175,16 +159,27 @@ def open_with_encoding(filename, mode='r', encoding=None, limit_byte_check=-1): newline='') # Preserve line endings +def _detect_magic_comment(filename: str): + try: + with open(filename) as input_file: + for idx, line in enumerate(input_file): + if idx >= 2: + break + match = ENCODING_MAGIC_COMMENT.search(line) + if match: + return match.groups()[0] + except Exception: + pass + # Python3's default encoding + return 'utf-8' + + def detect_encoding(filename, limit_byte_check=-1): """Return file encoding.""" + encoding = _detect_magic_comment(filename) try: - with open(filename, 'rb') as input_file: - from lib2to3.pgen2 import tokenize as lib2to3_tokenize - encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0] - with open_with_encoding(filename, encoding=encoding) as test_file: test_file.read(limit_byte_check) - return encoding except (LookupError, SyntaxError, UnicodeDecodeError): return 'latin-1' @@ -1717,69 +1712,6 @@ def split_and_strip_non_empty_lines(text): return [line.strip() for line in text.splitlines() if line.strip()] -def refactor(source, fixer_names, ignore=None, filename=''): - """Return refactored code using lib2to3. - - Skip if ignore string is produced in the refactored code. - - """ - not_found_end_of_file_newline = source and source.rstrip("\r\n") == source - if not_found_end_of_file_newline: - input_source = source + "\n" - else: - input_source = source - - from lib2to3 import pgen2 - try: - new_text = refactor_with_2to3(input_source, - fixer_names=fixer_names, - filename=filename) - except (pgen2.parse.ParseError, - SyntaxError, - UnicodeDecodeError, - UnicodeEncodeError): - return source - - if ignore: - if ignore in new_text and ignore not in source: - return source - - if not_found_end_of_file_newline: - return new_text.rstrip("\r\n") - - return new_text - - -def code_to_2to3(select, ignore, where='', verbose=False): - fixes = set() - for code, fix in CODE_TO_2TO3.items(): - if code_match(code, select=select, ignore=ignore): - if verbose: - print('---> Applying {} fix for {}'.format(where, - code.upper()), - file=sys.stderr) - fixes |= set(fix) - return fixes - - -def fix_2to3(source, - aggressive=True, select=None, ignore=None, filename='', - where='global', verbose=False): - """Fix various deprecated code (via lib2to3).""" - if not aggressive: - return source - - select = select or [] - ignore = ignore or [] - - return refactor(source, - code_to_2to3(select=select, - ignore=ignore, - where=where, - verbose=verbose), - filename=filename) - - def find_newline(source): """Return type of newline used in source. @@ -3175,24 +3107,6 @@ def _leading_space_count(line): return i -def refactor_with_2to3(source_text, fixer_names, filename=''): - """Use lib2to3 to refactor the source. - - Return the refactored source code. - - """ - from lib2to3.refactor import RefactoringTool - fixers = ['lib2to3.fixes.fix_' + name for name in fixer_names] - tool = RefactoringTool(fixer_names=fixers, explicit=fixers) - - from lib2to3.pgen2 import tokenize as lib2to3_tokenize - try: - # The name parameter is necessary particularly for the "import" fixer. - return str(tool.refactor_string(source_text, name=filename)) - except lib2to3_tokenize.TokenError: - return source_text - - def check_syntax(code): """Return True if syntax is okay.""" try: @@ -3685,14 +3599,6 @@ def apply_global_fixes(source, options, where='global', filename='', source = function(source, aggressive=options.aggressive) - source = fix_2to3(source, - aggressive=options.aggressive, - select=options.select, - ignore=options.ignore, - filename=filename, - where=where, - verbose=options.verbose) - return source @@ -4127,10 +4033,6 @@ def supported_fixes(): yield (code.upper() + (4 - len(code)) * ' ', re.sub(r'\s+', ' ', docstring_summary(function.__doc__))) - for code in sorted(CODE_TO_2TO3): - yield (code.upper() + (4 - len(code)) * ' ', - re.sub(r'\s+', ' ', docstring_summary(fix_2to3.__doc__))) - def docstring_summary(docstring): """Return summary of docstring.""" diff --git a/test/test_autopep8.py b/test/test_autopep8.py index bbee8ccd..4040ed42 100755 --- a/test/test_autopep8.py +++ b/test/test_autopep8.py @@ -384,37 +384,6 @@ def test_split_at_offsets_with_out_of_order(self): self.assertEqual(['12', '3', '4'], autopep8.split_at_offsets('1234', [3, 2])) - def test_fix_2to3(self): - self.assertEqual( - 'try: pass\nexcept ValueError as e: pass\n', - autopep8.fix_2to3('try: pass\nexcept ValueError, e: pass\n')) - - self.assertEqual( - 'while True: pass\n', - autopep8.fix_2to3('while 1: pass\n')) - - self.assertEqual( - """\ -import sys -sys.maxsize -""", - autopep8.fix_2to3("""\ -import sys -sys.maxint -""")) - - def test_fix_2to3_subset(self): - line = 'type(res) == type(42)\n' - fixed = 'isinstance(res, type(42))\n' - - self.assertEqual(fixed, autopep8.fix_2to3(line)) - self.assertEqual(fixed, autopep8.fix_2to3(line, select=['E721'])) - self.assertEqual(fixed, autopep8.fix_2to3(line, select=['E7'])) - - self.assertEqual(line, autopep8.fix_2to3(line, select=['W'])) - self.assertEqual(line, autopep8.fix_2to3(line, select=['E999'])) - self.assertEqual(line, autopep8.fix_2to3(line, ignore=['E721'])) - def test_is_python_file(self): self.assertTrue(autopep8.is_python_file( os.path.join(ROOT_DIR, 'autopep8.py'))) From 328b352fd56882d671caa9d73c2ea8d1a226c301 Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sat, 3 Dec 2022 11:34:32 +0900 Subject: [PATCH 2/9] impl: e721 fixed method --- autopep8.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/autopep8.py b/autopep8.py index 6d27445d..091f83e3 100755 --- a/autopep8.py +++ b/autopep8.py @@ -86,7 +86,7 @@ class documentation for more information. from configparser import ConfigParser as SafeConfigParser, Error import pycodestyle -from pycodestyle import STARTSWITH_INDENT_STATEMENT_REGEX +from pycodestyle import STARTSWITH_INDENT_STATEMENT_REGEX, COMPARE_TYPE_REGEX __version__ = '2.0.4' @@ -444,7 +444,7 @@ class FixPEP8(object): - e502 - e701,e702,e703,e704 - e711,e712,e713,e714 - - e722 + - e721,e722 - e731 - w291 - w503,504 @@ -1257,6 +1257,24 @@ def fix_e714(self, result): new_target[:pos_start], 'is not', new_target[pos_end:]) self.source[line_index] = new_target + def fix_e721(self, result): + """fix comparison type""" + (line_index, _, target) = get_index_offset_contents(result, + self.source) + match = COMPARE_TYPE_REGEX.search(target) + if match: + start = match.start() + end = match.end() + _prefix = "" + _type_comp = f"{match.groups()[0]}, {target[:start]}" + _tmp = target[:start].split() + if len(_tmp) >= 2: + _type_comp = f"{match.groups()[0]}, {_tmp[-1]}" + _prefix = "".join(_tmp[:-1]) + + fix_line = f"{_prefix} isinstance({_type_comp}){target[end:]}" + self.source[line_index] = fix_line + def fix_e722(self, result): """fix bare except""" (line_index, _, target) = get_index_offset_contents(result, From a7ada1ff0272fd77206941cb6a7c4be9a53a995d Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 13:05:37 +0900 Subject: [PATCH 3/9] fix: e721 --- autopep8.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/autopep8.py b/autopep8.py index 091f83e3..5df1a1bd 100755 --- a/autopep8.py +++ b/autopep8.py @@ -1266,13 +1266,21 @@ def fix_e721(self, result): start = match.start() end = match.end() _prefix = "" - _type_comp = f"{match.groups()[0]}, {target[:start]}" + if match.groups()[0] is None: + _target_obj = match.groups()[1] + else: + _target_obj = match.groups()[0] + + _type_comp = f"{_target_obj}, {target[:start]}" _tmp = target[:start].split() if len(_tmp) >= 2: - _type_comp = f"{match.groups()[0]}, {_tmp[-1]}" + _type_comp = f"{_target_obj}, {_tmp[-1]}" _prefix = "".join(_tmp[:-1]) - fix_line = f"{_prefix} isinstance({_type_comp}){target[end:]}" + if match.groups()[0] is None: + fix_line = f"{_prefix} isinstance({_type_comp}{target[end:-1]}){target[-1:]}" + else: + fix_line = f"{_prefix} isinstance({_type_comp}){target[end:]}" self.source[line_index] = fix_line def fix_e722(self, result): From a423a616102155d870cee2d9674ce9fdd9c8bf6b Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 13:09:20 +0900 Subject: [PATCH 4/9] strict lint --- autopep8.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autopep8.py b/autopep8.py index 5df1a1bd..76004856 100755 --- a/autopep8.py +++ b/autopep8.py @@ -1278,7 +1278,9 @@ def fix_e721(self, result): _prefix = "".join(_tmp[:-1]) if match.groups()[0] is None: - fix_line = f"{_prefix} isinstance({_type_comp}{target[end:-1]}){target[-1:]}" + _suffix = target[-1:] + _target = f"{_type_comp}{target[end:-1]}" + fix_line = f"{_prefix} isinstance({_target}){_suffix}" else: fix_line = f"{_prefix} isinstance({_type_comp}){target[end:]}" self.source[line_index] = fix_line From f785f8ad1ffb727adb8d4304843675565bdccf5d Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 19:04:31 +0900 Subject: [PATCH 5/9] fix: detect encoding for utf-8 with bom --- autopep8.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/autopep8.py b/autopep8.py index 76004856..ee2175da 100755 --- a/autopep8.py +++ b/autopep8.py @@ -159,10 +159,12 @@ def open_with_encoding(filename, mode='r', encoding=None, limit_byte_check=-1): newline='') # Preserve line endings -def _detect_magic_comment(filename: str): +def _detect_encoding_from_file(filename: str): try: with open(filename) as input_file: for idx, line in enumerate(input_file): + if idx == 0 and line[0] == '\ufeff': + return "utf-8-sig" if idx >= 2: break match = ENCODING_MAGIC_COMMENT.search(line) @@ -176,7 +178,9 @@ def _detect_magic_comment(filename: str): def detect_encoding(filename, limit_byte_check=-1): """Return file encoding.""" - encoding = _detect_magic_comment(filename) + encoding = _detect_encoding_from_file(filename) + if encoding == "utf-8-sig": + return encoding try: with open_with_encoding(filename, encoding=encoding) as test_file: test_file.read(limit_byte_check) From 5eed9cce4742fbdf8d8961cd11d47aa56a7cdf6d Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 19:08:36 +0900 Subject: [PATCH 6/9] impl: e721 --- autopep8.py | 59 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/autopep8.py b/autopep8.py index ee2175da..00b1bc3b 100755 --- a/autopep8.py +++ b/autopep8.py @@ -86,7 +86,7 @@ class documentation for more information. from configparser import ConfigParser as SafeConfigParser, Error import pycodestyle -from pycodestyle import STARTSWITH_INDENT_STATEMENT_REGEX, COMPARE_TYPE_REGEX +from pycodestyle import STARTSWITH_INDENT_STATEMENT_REGEX __version__ = '2.0.4' @@ -109,6 +109,11 @@ class documentation for more information. ENCODING_MAGIC_COMMENT = re.compile( r'^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)' ) +COMPARE_TYPE_REGEX = re.compile( + r'([=!]=)\s+type(?:\s*\(\s*([^)]*[^ )])\s*\))' + r'|\btype(?:\s*\(\s*([^)]*[^ )])\s*\))\s+([=!]=)' +) +TYPE_REGEX = re.compile(r'(type\s*\(\s*[^)]*?[^\s)]\s*\))') EXIT_CODE_OK = 0 EXIT_CODE_ERROR = 1 @@ -1267,26 +1272,50 @@ def fix_e721(self, result): self.source) match = COMPARE_TYPE_REGEX.search(target) if match: + # NOTE: match objects + # * type(a) == type(b) -> (None, None, 'a', '==') + # * str == type(b) -> ('==', 'b', None, None) + # * type("") != type(b) -> (None, None, '""', '!=') start = match.start() end = match.end() - _prefix = "" - if match.groups()[0] is None: - _target_obj = match.groups()[1] + first_match_type_obj = match.groups()[1] + if first_match_type_obj is None: + _target_obj = match.groups()[2] else: - _target_obj = match.groups()[0] + _target_obj = match.groups()[1] + isinstance_stmt = "isinstance" + is_not_condition = match.groups()[0] == "!=" or match.groups()[3] == "!=" + if is_not_condition: + isinstance_stmt = "not isinstance" + + _prefix = "" + _suffix = "" _type_comp = f"{_target_obj}, {target[:start]}" - _tmp = target[:start].split() - if len(_tmp) >= 2: - _type_comp = f"{_target_obj}, {_tmp[-1]}" - _prefix = "".join(_tmp[:-1]) - - if match.groups()[0] is None: - _suffix = target[-1:] - _target = f"{_type_comp}{target[end:-1]}" - fix_line = f"{_prefix} isinstance({_target}){_suffix}" + + _prefix_tmp = target[:start].split() + if len(_prefix_tmp) >= 1: + _type_comp = f"{_target_obj}, {target[:start]}" + if first_match_type_obj is not None: + _prefix = " ".join(_prefix_tmp[:-1]) + _type_comp = f"{_target_obj}, {_prefix_tmp[-1]}" + else: + _prefix = " ".join(_prefix_tmp) + + _suffix_tmp = target[end:] + _suffix_type_match = TYPE_REGEX.search(_suffix_tmp) + if len(_suffix_tmp.split()) >= 1 and _suffix_type_match: + if _suffix_type_match: + type_match_end = _suffix_type_match.end() + _suffix = _suffix_tmp[type_match_end:] + if _suffix_type_match: + cmp_b = _suffix_type_match.groups()[0] + _type_comp = f"{_target_obj}, {cmp_b}" + + if first_match_type_obj is None: + fix_line = f"{_prefix} {isinstance_stmt}({_type_comp}){_suffix}" else: - fix_line = f"{_prefix} isinstance({_type_comp}){target[end:]}" + fix_line = f"{_prefix} {isinstance_stmt}({_type_comp}){target[end:]}" self.source[line_index] = fix_line def fix_e722(self, result): From 2ef7c186cf1c5d0ff52e79a119c103f23f7843eb Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 19:09:09 +0900 Subject: [PATCH 7/9] add unit tests for e721 --- test/suite/E72.py | 4 ++-- test/suite/out/E72.py | 10 +++++----- test/test_autopep8.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/test/suite/E72.py b/test/suite/E72.py index 8eb34cb3..d4e6df73 100644 --- a/test/suite/E72.py +++ b/test/suite/E72.py @@ -26,9 +26,9 @@ assert type(res) == type((0)) #: E721 assert type(res) != type((1, )) -#: E721 +#: Okay assert type(res) is type((1, )) -#: E721 +#: Okay assert type(res) is not type((1, )) #: E211 E721 assert type(res) == type ([2, ]) diff --git a/test/suite/out/E72.py b/test/suite/out/E72.py index df5cc0d3..f84724c4 100644 --- a/test/suite/out/E72.py +++ b/test/suite/out/E72.py @@ -12,7 +12,7 @@ #: E721 import types -if not isinstance(res, types.ListType): +if type(res) is not types.ListType: pass #: E721 assert isinstance(res, type(False)) or isinstance(res, type(None)) @@ -26,10 +26,10 @@ assert isinstance(res, type((0))) #: E721 assert not isinstance(res, type((1, ))) -#: E721 -assert isinstance(res, type((1, ))) -#: E721 -assert not isinstance(res, type((1, ))) +#: Okay +assert type(res) is type((1, )) +#: Okay +assert type(res) is not type((1, )) #: E211 E721 assert isinstance(res, type([2, ])) #: E201 E201 E202 E721 diff --git a/test/test_autopep8.py b/test/test_autopep8.py index 4040ed42..64031066 100755 --- a/test/test_autopep8.py +++ b/test/test_autopep8.py @@ -4375,6 +4375,24 @@ def test_e721_in_conditional(self): with autopep8_context(line, options=['--aggressive']) as result: self.assertEqual(fixed, result) + def test_e721_in_conditional_pat2(self): + line = "if type(res) == type(42):\n pass\n" + fixed = "if isinstance(res, type(42)):\n pass\n" + with autopep8_context(line, options=['--aggressive']) as result: + self.assertEqual(fixed, result) + + def test_e721_in_not_conditional(self): + line = "if type(res) != type(''):\n pass\n" + fixed = "if not isinstance(res, type('')):\n pass\n" + with autopep8_context(line, options=['--aggressive']) as result: + self.assertEqual(fixed, result) + + def test_e721_in_not_conditional_pat2(self): + line = "if type(a) != type(b) or type(a) == type(ccc):\n pass\n" + fixed = "if not isinstance(a, type(b)) or isinstance(a, type(ccc)):\n pass\n" + with autopep8_context(line, options=['--aggressive']) as result: + self.assertEqual(fixed, result) + def test_e722(self): line = "try:\n print(a)\nexcept:\n pass\n" fixed = "try:\n print(a)\nexcept BaseException:\n pass\n" From 3c874e4e1bcf2cbefbcea435c8b54f14f53720ba Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 19:09:25 +0900 Subject: [PATCH 8/9] update ignorefile --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3dea2851..2dcf75a1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ coverage.xml dist htmlcov pep8.py +test/suite/out/*.py.err From bc214807f1ac31a7d6f2f5248d944cd963d5a52f Mon Sep 17 00:00:00 2001 From: Hideo Hattori Date: Sun, 17 Mar 2024 19:16:31 +0900 Subject: [PATCH 9/9] refactoring --- autopep8.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/autopep8.py b/autopep8.py index 00b1bc3b..80b6ec12 100755 --- a/autopep8.py +++ b/autopep8.py @@ -1278,19 +1278,22 @@ def fix_e721(self, result): # * type("") != type(b) -> (None, None, '""', '!=') start = match.start() end = match.end() + _prefix = "" + _suffix = "" first_match_type_obj = match.groups()[1] if first_match_type_obj is None: _target_obj = match.groups()[2] else: _target_obj = match.groups()[1] + _suffix = target[end:] - isinstance_stmt = "isinstance" - is_not_condition = match.groups()[0] == "!=" or match.groups()[3] == "!=" + isinstance_stmt = " isinstance" + is_not_condition = ( + match.groups()[0] == "!=" or match.groups()[3] == "!=" + ) if is_not_condition: - isinstance_stmt = "not isinstance" + isinstance_stmt = " not isinstance" - _prefix = "" - _suffix = "" _type_comp = f"{_target_obj}, {target[:start]}" _prefix_tmp = target[:start].split() @@ -1312,10 +1315,7 @@ def fix_e721(self, result): cmp_b = _suffix_type_match.groups()[0] _type_comp = f"{_target_obj}, {cmp_b}" - if first_match_type_obj is None: - fix_line = f"{_prefix} {isinstance_stmt}({_type_comp}){_suffix}" - else: - fix_line = f"{_prefix} {isinstance_stmt}({_type_comp}){target[end:]}" + fix_line = f"{_prefix}{isinstance_stmt}({_type_comp}){_suffix}" self.source[line_index] = fix_line def fix_e722(self, result):