Skip to content

Commit

Permalink
Allow type ignores after type comments (#6591)
Browse files Browse the repository at this point in the history 
This is a hack in fastparse, but allows for the following to pass typechecking:

`x = 1 # type: str # type: ignore`

This also handles the edge case where there is a `# type: ignore` in a comment, which we don't want to pick up. See the tests for more examples.

Fixes #5967
  • Loading branch information
@ethanhs
ethanhs committed Mar 27, 2019
1 parent affb032 commit d1e597d
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 30 deletions.
56 changes: 38 additions & 18 deletions mypy/fastparse.py
View file
@@ -1,3 +1,4 @@
import re
import sys

from typing import (
Expand Down Expand Up @@ -120,6 +121,8 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str,

TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final

TYPE_IGNORE_PATTERN = re.compile(r'[^#]*#\s*type:\s*ignore\s*($|#)')


# Older versions of typing don't allow using overload outside stubs,
# so provide a dummy.
Expand Down Expand Up @@ -180,18 +183,19 @@ def parse_type_comment(type_comment: str,
line: int,
errors: Optional[Errors],
assume_str_is_unicode: bool = True,
) -> Optional[Type]:
) -> Tuple[bool, Optional[Type]]:
try:
typ = ast3_parse(type_comment, '<type_comment>', 'eval')
except SyntaxError as e:
if errors is not None:
errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True)
return None
return False, None
else:
raise
else:
extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) is not None
assert isinstance(typ, ast3_Expression)
return TypeConverter(errors, line=line,
return extra_ignore, TypeConverter(errors, line=line,
assume_str_is_unicode=assume_str_is_unicode).visit(typ.body)


Expand All @@ -212,8 +216,8 @@ def parse_type_string(expr_string: str, expr_fallback_name: str,
code with unicode_literals...) and setting `assume_str_is_unicode` accordingly.
"""
try:
node = parse_type_comment(expr_string.strip(), line=line, errors=None,
assume_str_is_unicode=assume_str_is_unicode)
_, node = parse_type_comment(expr_string.strip(), line=line, errors=None,
assume_str_is_unicode=assume_str_is_unicode)
if isinstance(node, UnboundType) and node.original_str_expr is None:
node.original_str_expr = expr_string
node.original_str_fallback = expr_fallback_name
Expand Down Expand Up @@ -247,6 +251,8 @@ def __init__(self,
self.is_stub = is_stub
self.errors = errors

self.extra_type_ignores = [] # type: List[int]

# Cache of visit_X methods keyed by type of visited object
self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]]

Expand Down Expand Up @@ -389,11 +395,12 @@ def translate_module_id(self, id: str) -> str:

def visit_Module(self, mod: ast3.Module) -> MypyFile:
body = self.fix_function_overloads(self.translate_stmt_list(mod.body))

ignores = [ti.lineno for ti in mod.type_ignores]
ignores.extend(self.extra_type_ignores)
return MypyFile(body,
self.imports,
False,
{ti.lineno for ti in mod.type_ignores},
{*ignores},
)

# --- stmt ---
Expand Down Expand Up @@ -587,7 +594,10 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int,
if annotation is not None:
arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation)
elif type_comment is not None:
arg_type = parse_type_comment(type_comment, arg.lineno, self.errors)
extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(arg.lineno)

return Argument(Var(arg.arg), arg_type, self.visit(default), kind)

def fail_arg(self, msg: str, arg: ast3.arg) -> None:
Expand Down Expand Up @@ -642,7 +652,9 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt:
lvalues = self.translate_expr_list(n.targets)
rvalue = self.visit(n.value)
if n.type_comment is not None:
typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
typ = None
s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False)
Expand Down Expand Up @@ -674,7 +686,9 @@ def visit_NamedExpr(self, n: NamedExpr) -> None:
# For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_For(self, n: ast3.For) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = ForStmt(self.visit(n.target),
Expand All @@ -687,7 +701,9 @@ def visit_For(self, n: ast3.For) -> ForStmt:
# AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = ForStmt(self.visit(n.target),
Expand Down Expand Up @@ -716,7 +732,9 @@ def visit_If(self, n: ast3.If) -> IfStmt:
# With(withitem* items, stmt* body, string? type_comment)
def visit_With(self, n: ast3.With) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
node = WithStmt([self.visit(i.context_expr) for i in n.items],
Expand All @@ -728,7 +746,9 @@ def visit_With(self, n: ast3.With) -> WithStmt:
# AsyncWith(withitem* items, stmt* body, string? type_comment)
def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
s = WithStmt([self.visit(i.context_expr) for i in n.items],
Expand Down Expand Up @@ -1211,11 +1231,11 @@ def visit_raw_str(self, s: str) -> Type:
# An escape hatch that allows the AST walker in fastparse2 to
# directly hook into the Python 3.5 type converter in some cases
# without needing to create an intermediary `Str` object.
return (parse_type_comment(s.strip(),
self.line,
self.errors,
self.assume_str_is_unicode)
or AnyType(TypeOfAny.from_error))
_, typ = parse_type_comment(s.strip(),
self.line,
self.errors,
self.assume_str_is_unicode)
return typ or AnyType(TypeOfAny.from_error)

def visit_Call(self, e: Call) -> Type:
# Parse the arg constructor
Expand Down
33 changes: 21 additions & 12 deletions mypy/fastparse2.py
View file
Expand Up @@ -162,6 +162,8 @@ def __init__(self,
# Cache of visit_X methods keyed by type of visited object
self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]]

self.extra_type_ignores = [] # type: List[int]

def fail(self, msg: str, line: int, column: int) -> None:
self.errors.report(line, column, msg, blocker=True)

Expand Down Expand Up @@ -301,11 +303,12 @@ def translate_module_id(self, id: str) -> str:

def visit_Module(self, mod: ast27.Module) -> MypyFile:
body = self.fix_function_overloads(self.translate_stmt_list(mod.body))

ignores = [ti.lineno for ti in mod.type_ignores]
ignores.extend(self.extra_type_ignores)
return MypyFile(body,
self.imports,
False,
{ti.lineno for ti in mod.type_ignores},
{*ignores},
)

# --- stmt ---
Expand Down Expand Up @@ -543,8 +546,10 @@ def visit_Delete(self, n: ast27.Delete) -> DelStmt:
def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt:
typ = None
if n.type_comment:
typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)

stmt = AssignmentStmt(self.translate_expr_list(n.targets),
self.visit(n.value),
Expand All @@ -561,15 +566,17 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt:
# For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
def visit_For(self, n: ast27.For) -> ForStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
typ = None
stmt = ForStmt(self.visit(n.target),
self.visit(n.iter),
self.as_required_block(n.body, n.lineno),
self.as_block(n.orelse, n.lineno),
target_type)
typ)
return self.set_line(stmt, n)

# While(expr test, stmt* body, stmt* orelse)
Expand All @@ -589,14 +596,16 @@ def visit_If(self, n: ast27.If) -> IfStmt:
# With(withitem* items, stmt* body, string? type_comment)
def visit_With(self, n: ast27.With) -> WithStmt:
if n.type_comment is not None:
target_type = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
assume_str_is_unicode=self.unicode_literals)
if extra_ignore:
self.extra_type_ignores.append(n.lineno)
else:
target_type = None
typ = None
stmt = WithStmt([self.visit(n.context_expr)],
[self.visit(n.optional_vars)],
self.as_required_block(n.body, n.lineno),
target_type)
typ)
return self.set_line(stmt, n)

def visit_Raise(self, n: ast27.Raise) -> RaiseStmt:
Expand Down
29 changes: 29 additions & 0 deletions test-data/unit/check-fastparse.test
View file
Expand Up @@ -88,6 +88,35 @@ def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation
def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation
def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation

[case testFastParseTypeWithIgnore]
def f(x, # type: x # type: ignore
):
# type: (...) -> None
pass

[case testFastParseVariableTypeWithIgnore]

x = 1 # type: str # type: ignore

[case testFastParseVariableTypeWithIgnoreNoSpace]

x = 1 # type: str #type:ignore

[case testFastParseVariableTypeWithIgnoreAndComment]

x = 1 # type: str # type: ignore # comment

[case testFastParseTypeWithIgnoreWithStmt]
with open('test', 'r') as f: # type: int # type: ignore
pass

[case testFastParseTypeWithIgnoreForStmt]
for i in (1, 2, 3, 100): # type: str # type: ignore
pass

[case testFastParseVariableCommentThenIgnore]
a="test" # type: int #comment # type: ignore # E: Incompatible types in assignment (expression has type "str", variable has type "int")

[case testFastParseProperty]

class C:
Expand Down

0 comments on commit d1e597d

Please sign in to comment.