From d1e597dcd90d10b296b246f2df044517c1e7673c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 26 Mar 2019 21:06:06 -0700 Subject: [PATCH] Allow type ignores after type comments (#6591) This is a hack in fastparse, but allows for the following to pass typechecking: `x = 1 # type: str # type: ignore` This also handles the edge case where there is a `# type: ignore` in a comment, which we don't want to pick up. See the tests for more examples. Fixes #5967 --- mypy/fastparse.py | 56 +++++++++++++++++++---------- mypy/fastparse2.py | 33 ++++++++++------- test-data/unit/check-fastparse.test | 29 +++++++++++++++ 3 files changed, 88 insertions(+), 30 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 9cad8eedf174..63a2ac1095ae 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,3 +1,4 @@ +import re import sys from typing import ( @@ -120,6 +121,8 @@ def ast3_parse(source: Union[str, bytes], filename: str, mode: str, TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final +TYPE_IGNORE_PATTERN = re.compile(r'[^#]*#\s*type:\s*ignore\s*($|#)') + # Older versions of typing don't allow using overload outside stubs, # so provide a dummy. @@ -180,18 +183,19 @@ def parse_type_comment(type_comment: str, line: int, errors: Optional[Errors], assume_str_is_unicode: bool = True, - ) -> Optional[Type]: + ) -> Tuple[bool, Optional[Type]]: try: typ = ast3_parse(type_comment, '', 'eval') except SyntaxError as e: if errors is not None: errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True) - return None + return False, None else: raise else: + extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) is not None assert isinstance(typ, ast3_Expression) - return TypeConverter(errors, line=line, + return extra_ignore, TypeConverter(errors, line=line, assume_str_is_unicode=assume_str_is_unicode).visit(typ.body) @@ -212,8 +216,8 @@ def parse_type_string(expr_string: str, expr_fallback_name: str, code with unicode_literals...) and setting `assume_str_is_unicode` accordingly. """ try: - node = parse_type_comment(expr_string.strip(), line=line, errors=None, - assume_str_is_unicode=assume_str_is_unicode) + _, node = parse_type_comment(expr_string.strip(), line=line, errors=None, + assume_str_is_unicode=assume_str_is_unicode) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string node.original_str_fallback = expr_fallback_name @@ -247,6 +251,8 @@ def __init__(self, self.is_stub = is_stub self.errors = errors + self.extra_type_ignores = [] # type: List[int] + # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] @@ -389,11 +395,12 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast3.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - + ignores = [ti.lineno for ti in mod.type_ignores] + ignores.extend(self.extra_type_ignores) return MypyFile(body, self.imports, False, - {ti.lineno for ti in mod.type_ignores}, + {*ignores}, ) # --- stmt --- @@ -587,7 +594,10 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int, if annotation is not None: arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation) elif type_comment is not None: - arg_type = parse_type_comment(type_comment, arg.lineno, self.errors) + extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(arg.lineno) + return Argument(Var(arg.arg), arg_type, self.visit(default), kind) def fail_arg(self, msg: str, arg: ast3.arg) -> None: @@ -642,7 +652,9 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt: lvalues = self.translate_expr_list(n.targets) rvalue = self.visit(n.value) if n.type_comment is not None: - typ = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: typ = None s = AssignmentStmt(lvalues, rvalue, type=typ, new_syntax=False) @@ -674,7 +686,9 @@ def visit_NamedExpr(self, n: NamedExpr) -> None: # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast3.For) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -687,7 +701,9 @@ def visit_For(self, n: ast3.For) -> ForStmt: # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = ForStmt(self.visit(n.target), @@ -716,7 +732,9 @@ def visit_If(self, n: ast3.If) -> IfStmt: # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast3.With) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None node = WithStmt([self.visit(i.context_expr) for i in n.items], @@ -728,7 +746,9 @@ def visit_With(self, n: ast3.With) -> WithStmt: # AsyncWith(withitem* items, stmt* body, string? type_comment) def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: target_type = None s = WithStmt([self.visit(i.context_expr) for i in n.items], @@ -1211,11 +1231,11 @@ def visit_raw_str(self, s: str) -> Type: # An escape hatch that allows the AST walker in fastparse2 to # directly hook into the Python 3.5 type converter in some cases # without needing to create an intermediary `Str` object. - return (parse_type_comment(s.strip(), - self.line, - self.errors, - self.assume_str_is_unicode) - or AnyType(TypeOfAny.from_error)) + _, typ = parse_type_comment(s.strip(), + self.line, + self.errors, + self.assume_str_is_unicode) + return typ or AnyType(TypeOfAny.from_error) def visit_Call(self, e: Call) -> Type: # Parse the arg constructor diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index 365edeb00048..6865fd85a13a 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -162,6 +162,8 @@ def __init__(self, # Cache of visit_X methods keyed by type of visited object self.visitor_cache = {} # type: Dict[type, Callable[[Optional[AST]], Any]] + self.extra_type_ignores = [] # type: List[int] + def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg, blocker=True) @@ -301,11 +303,12 @@ def translate_module_id(self, id: str) -> str: def visit_Module(self, mod: ast27.Module) -> MypyFile: body = self.fix_function_overloads(self.translate_stmt_list(mod.body)) - + ignores = [ti.lineno for ti in mod.type_ignores] + ignores.extend(self.extra_type_ignores) return MypyFile(body, self.imports, False, - {ti.lineno for ti in mod.type_ignores}, + {*ignores}, ) # --- stmt --- @@ -543,8 +546,10 @@ def visit_Delete(self, n: ast27.Delete) -> DelStmt: def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: typ = None if n.type_comment: - typ = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) stmt = AssignmentStmt(self.translate_expr_list(n.targets), self.visit(n.value), @@ -561,15 +566,17 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) def visit_For(self, n: ast27.For) -> ForStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: - target_type = None + typ = None stmt = ForStmt(self.visit(n.target), self.visit(n.iter), self.as_required_block(n.body, n.lineno), self.as_block(n.orelse, n.lineno), - target_type) + typ) return self.set_line(stmt, n) # While(expr test, stmt* body, stmt* orelse) @@ -589,14 +596,16 @@ def visit_If(self, n: ast27.If) -> IfStmt: # With(withitem* items, stmt* body, string? type_comment) def visit_With(self, n: ast27.With) -> WithStmt: if n.type_comment is not None: - target_type = parse_type_comment(n.type_comment, n.lineno, self.errors, - assume_str_is_unicode=self.unicode_literals) + extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors, + assume_str_is_unicode=self.unicode_literals) + if extra_ignore: + self.extra_type_ignores.append(n.lineno) else: - target_type = None + typ = None stmt = WithStmt([self.visit(n.context_expr)], [self.visit(n.optional_vars)], self.as_required_block(n.body, n.lineno), - target_type) + typ) return self.set_line(stmt, n) def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index f3f5403ffc35..4815f1f45d87 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -88,6 +88,35 @@ def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation +[case testFastParseTypeWithIgnore] +def f(x, # type: x # type: ignore + ): + # type: (...) -> None + pass + +[case testFastParseVariableTypeWithIgnore] + +x = 1 # type: str # type: ignore + +[case testFastParseVariableTypeWithIgnoreNoSpace] + +x = 1 # type: str #type:ignore + +[case testFastParseVariableTypeWithIgnoreAndComment] + +x = 1 # type: str # type: ignore # comment + +[case testFastParseTypeWithIgnoreWithStmt] +with open('test', 'r') as f: # type: int # type: ignore + pass + +[case testFastParseTypeWithIgnoreForStmt] +for i in (1, 2, 3, 100): # type: str # type: ignore + pass + +[case testFastParseVariableCommentThenIgnore] +a="test" # type: int #comment # type: ignore # E: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testFastParseProperty] class C: