Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow type ignores after type comments #6591

Merged
merged 4 commits into from Mar 27, 2019
Merged

Allow type ignores after type comments #6591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
038b122
Allow type ignores after type comments
ethanhs Mar 26, 2019
223de15
Don't pick up type ignores in comments
ethanhs Mar 26, 2019
5c8d5cf
Merge branch 'master' of github.com:python/mypy into ignoretypecomment
ethanhs Mar 27, 2019
d0cf678
Use better regex to recognize extra type ignore, fix indents
ethanhs Mar 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
56 changes: 38 additions & 18 deletions mypy/fastparse.py
View file
Open in desktop
@@ -1,3 +1,4 @@
import re
import sys

from typing import (
Expand Down Expand Up @@ -120,6 +121,8 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str,

TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final

TYPE_IGNORE_PATTERN = re.compile(r'[^#]*#\s*type:\s*ignore\s*($|#)')


# Older versions of typing don't allow using overload outside stubs,
# so provide a dummy.
Expand Down Expand Up @@ -180,18 +183,19 @@ def parse_type_comment(type_comment: str,
line: int,
errors: Optional[Errors],
assume_str_is_unicode: bool = True,
) -> Optional[Type]:
) -> Tuple[bool, Optional[Type]]:
try:
typ = ast3_parse(type_comment, '<type_comment>', 'eval')
except SyntaxError as e:
if errors is not None:
errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True)
return None
return False, None
else:
raise
else:
extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) is not None
assert isinstance(typ, ast3_Expression)
return TypeConverter(errors, line=line,
return extra_ignore, TypeConverter(errors, line=line,
assume_str_is_unicode=assume_str_is_unicode).visit(typ.body)


Expand All @@ -212,8 +216,8 @@ def parse_type_string(expr_string: str, expr_fallback_name: str,
code with unicode_literals...) and setting `assume_str_is_unicode` accordingly.
"""
try:
node = parse_type_comment(expr_string.strip(), line=line, errors=None,
assume_str_is_unicode=assume_str_is_unicode)
_, node = parse_type_comment(expr_string.strip(), line=line, errors=None,
assume_str_is_unicode=assume_str_is_unicode)
if isinstance(node, UnboundType) and node.original_str_expr is None:
node.original_str_expr = expr_string
node.original_str_fallback = expr_fallback_name
Expand Down Expand Up @@ -247,6 +251,8 @@ def __init__(self,
self.is_stub = is_stub
self.errors = errors

self.extra_type_ignores = [] # type: List[int]

# Cache of visit_X methods keyed by type of visited object
self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]]

Expand Down Expand Up @@ -389,11 +395,12 @@ def translate_module_id(self, id: str) -> str:

def visit_Module(self, mod: ast3.Module) -> MypyFile:
body = self.fix_function_overloads(self.translate_stmt_list(mod.body))

ignores = [ti.lineno for ti in mod.type_ignores]
ignores.extend(self.extra_type_ignores)
return MypyFile(body,
self.imports,
False,
{ti.lineno for ti in mod.type_ignores},
{*ignores},
)

# --- stmt ---
Expand Down Expand Up @@ -587,7 +594,10 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int,
if annotation is not None:
arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation)
elif type_comment is not None:
arg_type = parse_type_comment(type_comment, arg.lineno, self.errors)
extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(arg.lineno)

return Argument(Var(arg.arg), arg_type, self.visit(default), kind)

def fail_arg(self, msg: str, arg: ast3.arg) -> None:
Expand Down Expand Up @@ -642,7 +652,9 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt:
lvalues = self.translate_expr_list(n.targets)
rvalue = self.visit(n.value)
if n.type_comment is not None:
typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
typ = None
s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False)
Expand Down Expand Up @@ -674,7 +686,9 @@ def visit_NamedExpr(self, n: NamedExpr) -> None:
# For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_For(self, n: ast3.For) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = ForStmt(self.visit(n.target),
Expand All @@ -687,7 +701,9 @@ def visit_For(self, n: ast3.For) -> ForStmt:
# AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = ForStmt(self.visit(n.target),
Expand Down Expand Up @@ -716,7 +732,9 @@ def visit_If(self, n: ast3.If) -> IfStmt:
# With(withitem* items, stmt* body, string? type_comment)
def visit_With(self, n: ast3.With) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = WithStmt([self.visit(i.context_expr) for i in n.items],
Expand All @@ -728,7 +746,9 @@ def visit_With(self, n: ast3.With) -> WithStmt:
# AsyncWith(withitem* items, stmt* body, string? type_comment)
def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
s = WithStmt([self.visit(i.context_expr) for i in n.items],
Expand Down Expand Up @@ -1211,11 +1231,11 @@ def visit_raw_str(self, s: str) -> Type:
# An escape hatch that allows the AST walker in fastparse2 to
# directly hook into the Python 3.5 type converter in some cases
# without needing to create an intermediary `Str` object.
return (parse_type_comment(s.strip(),
self.line,
self.errors,
self.assume_str_is_unicode)
or AnyType(TypeOfAny.from_error))
_, typ = parse_type_comment(s.strip(),
self.line,
self.errors,
self.assume_str_is_unicode)
return typ or AnyType(TypeOfAny.from_error)

def visit_Call(self, e: Call) -> Type:
# Parse the arg constructor
Expand Down
33 changes: 21 additions & 12 deletions mypy/fastparse2.py
View file
Open in desktop
Expand Up @@ -162,6 +162,8 @@ def __init__(self,
# Cache of visit_X methods keyed by type of visited object
self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]]

self.extra_type_ignores = [] # type: List[int]

def fail(self, msg: str, line: int, column: int) -> None:
self.errors.report(line, column, msg, blocker=True)

Expand Down Expand Up @@ -301,11 +303,12 @@ def translate_module_id(self, id: str) -> str:

def visit_Module(self, mod: ast27.Module) -> MypyFile:
body = self.fix_function_overloads(self.translate_stmt_list(mod.body))

ignores = [ti.lineno for ti in mod.type_ignores]
ignores.extend(self.extra_type_ignores)
return MypyFile(body,
self.imports,
False,
{ti.lineno for ti in mod.type_ignores},
{*ignores},
)

# --- stmt ---
Expand Down Expand Up @@ -543,8 +546,10 @@ def visit_Delete(self, n: ast27.Delete) -> DelStmt:
def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt:
typ = None
if n.type_comment:
typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)

stmt = AssignmentStmt(self.translate_expr_list(n.targets),
self.visit(n.value),
Expand All @@ -561,15 +566,17 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt:
# For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_For(self, n: ast27.For) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
typ = None
stmt = ForStmt(self.visit(n.target),
self.visit(n.iter),
self.as_required_block(n.body, n.lineno),
self.as_block(n.orelse, n.lineno),
target_type)
typ)
return self.set_line(stmt, n)

# While(expr test, stmt* body, stmt* orelse)
Expand All @@ -589,14 +596,16 @@ def visit_If(self, n: ast27.If) -> IfStmt:
# With(withitem* items, stmt* body, string? type_comment)
def visit_With(self, n: ast27.With) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
typ = None
stmt = WithStmt([self.visit(n.context_expr)],
[self.visit(n.optional_vars)],
self.as_required_block(n.body, n.lineno),
target_type)
typ)
return self.set_line(stmt, n)

def visit_Raise(self, n: ast27.Raise) -> RaiseStmt:
Expand Down
29 changes: 29 additions & 0 deletions test-data/unit/check-fastparse.test
View file
Open in desktop
Expand Up @@ -88,6 +88,35 @@ def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation
def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation
def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation

[case testFastParseTypeWithIgnore]
def f(x, # type: x # type: ignore
):
# type: (...) -> None
pass

[case testFastParseVariableTypeWithIgnore]

x = 1 # type: str # type: ignore

[case testFastParseVariableTypeWithIgnoreNoSpace]

x = 1 # type: str #type:ignore

[case testFastParseVariableTypeWithIgnoreAndComment]

x = 1 # type: str # type: ignore # comment

[case testFastParseTypeWithIgnoreWithStmt]
with open('test', 'r') as f: # type: int # type: ignore
pass

[case testFastParseTypeWithIgnoreForStmt]
for i in (1, 2, 3, 100): # type: str # type: ignore
pass

[case testFastParseVariableCommentThenIgnore]
a="test" # type: int #comment # type: ignore # E: Incompatible types in assignment (expression has type "str", variable has type "int")

[case testFastParseProperty]

class C:
Expand Down