fastparse2.py

"""
This file is nearly identical to `fastparse.py`, except that it works with a Python 2
AST instead of a Python 3 AST.

Previously, how we handled Python 2 code was by first obtaining the Python 2 AST via
typed_ast, converting it into a Python 3 AST by using typed_ast.conversion, then
running it through mypy.fastparse.

While this worked, it did add some overhead, especially in larger Python 2 codebases.
This module allows us to skip the conversion step, saving us some time.

The reason why this file is not easily merged with mypy.fastparse despite the large amount
of redundancy is because the Python 2 AST and the Python 3 AST nodes belong to two completely
different class hierarchies, which made it difficult to write a shared visitor between the
two in a typesafe way.
"""
import sys
import warnings

import typing  # for typing.Type, which conflicts with types.Type
from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List
from typing_extensions import Final, Literal

from mypy.sharedparse import (
    special_function_elide_names, argument_elide_name,
)
from mypy.nodes import (
    MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef,
    ClassDef, Decorator, Block, Var, OperatorAssignmentStmt,
    ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt,
    DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
    WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt,
    TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr,
    DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, UnicodeExpr,
    FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr,
    UnaryExpr, LambdaExpr, ComparisonExpr, DictionaryComprehension,
    SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument,
    Expression, Statement, BackquoteExpr, PrintStmt, ExecStmt,
    ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart, check_arg_names,
    FakeInfo,
)
from mypy.types import (
    Type, CallableType, AnyType, UnboundType, EllipsisType, TypeOfAny, Instance,
    ProperType
)
from mypy import message_registry, errorcodes as codes
from mypy.errors import Errors
from mypy.fastparse import (
    TypeConverter, parse_type_comment, bytes_to_human_readable_repr, parse_type_ignore_tag,
    TYPE_IGNORE_PATTERN
)
from mypy.options import Options
from mypy.reachability import mark_block_unreachable

try:
    from typed_ast import ast27
    from typed_ast.ast27 import (
        AST,
        Call,
        Name,
        Attribute,
        Tuple as ast27_Tuple,
    )
    # Import ast3 from fastparse, which has special case for Python 3.8
    from mypy.fastparse import ast3, ast3_parse
except ImportError:
    try:
        from typed_ast import ast35  # type: ignore[attr-defined]  # noqa: F401
    except ImportError:
        print('The typed_ast package is not installed.\n'
              'You can install it with `python3 -m pip install typed-ast`.',
              file=sys.stderr)
    else:
        print('You need a more recent version of the typed_ast package.\n'
              'You can update to the latest version with '
              '`python3 -m pip install -U typed-ast`.',
              file=sys.stderr)
    sys.exit(1)

N = TypeVar('N', bound=Node)

# There is no way to create reasonable fallbacks at this stage,
# they must be patched later.
MISSING_FALLBACK = FakeInfo("fallback can't be filled out until semanal")  # type: Final
_dummy_fallback = Instance(MISSING_FALLBACK, [], -1)  # type: Final

TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment'  # type: Final
TYPE_COMMENT_AST_ERROR = 'invalid type comment'  # type: Final


def parse(source: Union[str, bytes],
          fnam: str,
          module: Optional[str],
          errors: Optional[Errors] = None,
          options: Optional[Options] = None) -> MypyFile:
    """Parse a source file, without doing any semantic analysis.

    Return the parse tree. If errors is not provided, raise ParseError
    on failure. Otherwise, use the errors object to report parse errors.
    """
    raise_on_error = False
    if errors is None:
        errors = Errors()
        raise_on_error = True
    if options is None:
        options = Options()
    errors.set_file(fnam, module)
    is_stub_file = fnam.endswith('.pyi')
    try:
        assert options.python_version[0] < 3 and not is_stub_file
        # Disable deprecation warnings about <>.
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            ast = ast27.parse(source, fnam, 'exec')
        tree = ASTConverter(options=options,
                            errors=errors,
                            ).visit(ast)
        assert isinstance(tree, MypyFile)
        tree.path = fnam
        tree.is_stub = is_stub_file
    except SyntaxError as e:
        errors.report(e.lineno, e.offset, e.msg, blocker=True, code=codes.SYNTAX)
        tree = MypyFile([], [], False, {})

    if raise_on_error and errors.is_errors():
        errors.raise_error()

    return tree


def is_no_type_check_decorator(expr: ast27.expr) -> bool:
    if isinstance(expr, Name):
        return expr.id == 'no_type_check'
    elif isinstance(expr, Attribute):
        if isinstance(expr.value, Name):
            return expr.value.id == 'typing' and expr.attr == 'no_type_check'
    return False


class ASTConverter:
    def __init__(self,
                 options: Options,
                 errors: Errors) -> None:
        # 'C' for class, 'F' for function
        self.class_and_function_stack = []  # type: List[Literal['C', 'F']]
        self.imports = []  # type: List[ImportBase]

        self.options = options
        self.errors = errors

        # Indicates whether this file is being parsed with unicode_literals enabled.
        # Note: typed_ast already naturally takes unicode_literals into account when
        # parsing so we don't have to worry when analyzing strings within this class.
        #
        # The only place where we use this field is when we call fastparse's TypeConverter
        # and any related methods. That class accepts a Python 3 AST instead of a Python 2
        # AST: as a result, it don't special-case the `unicode_literals` import and won't know
        # exactly whether to parse some string as bytes or unicode.
        #
        # This distinction is relevant mostly when handling Literal types -- Literal[u"foo"]
        # is not the same type as Literal[b"foo"], and Literal["foo"] could mean either the
        # former or the latter based on context.
        #
        # This field is set in the 'visit_ImportFrom' method: it's ok to delay computing it
        # because any `from __future__ import blah` import must be located at the top of the
        # file, with the exception of the docstring. This means we're guaranteed to correctly
        # set this field before we encounter any type hints.
        self.unicode_literals = False

        # Cache of visit_X methods keyed by type of visited object
        self.visitor_cache = {}  # type: Dict[type, Callable[[Optional[AST]], Any]]

        self.type_ignores = {}  # type: Dict[int, List[str]]

    def fail(self, msg: str, line: int, column: int, blocker: bool = True) -> None:
        if blocker or not self.options.ignore_errors:
            self.errors.report(line, column, msg, blocker=blocker, code=codes.SYNTAX)

    def visit(self, node: Optional[AST]) -> Any:  # same as in typed_ast stub
        if node is None:
            return None
        typeobj = type(node)
        visitor = self.visitor_cache.get(typeobj)
        if visitor is None:
            method = 'visit_' + node.__class__.__name__
            visitor = getattr(self, method)
            self.visitor_cache[typeobj] = visitor
        return visitor(node)

    def set_line(self, node: N, n: Union[ast27.expr, ast27.stmt]) -> N:
        node.line = n.lineno
        node.column = n.col_offset
        return node

    def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]:
        res = []  # type: List[Expression]
        for e in l:
            exp = self.visit(e)
            assert isinstance(exp, Expression)
            res.append(exp)
        return res

    def get_lineno(self, node: Union[ast27.expr, ast27.stmt]) -> int:
        if isinstance(node, (ast27.ClassDef, ast27.FunctionDef)) and node.decorator_list:
            return node.decorator_list[0].lineno
        return node.lineno

    def translate_stmt_list(self,
                            stmts: Sequence[ast27.stmt],
                            module: bool = False) -> List[Statement]:
        # A "# type: ignore" comment before the first statement of a module
        # ignores the whole module:
        if (module and stmts and self.type_ignores
                and min(self.type_ignores) < self.get_lineno(stmts[0])):
            self.errors.used_ignored_lines[self.errors.file].add(min(self.type_ignores))
            block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
            mark_block_unreachable(block)
            return [block]

        res = []  # type: List[Statement]
        for stmt in stmts:
            node = self.visit(stmt)
            assert isinstance(node, Statement)
            res.append(node)
        return res

    def translate_type_comment(self, n: ast27.stmt,
                               type_comment: Optional[str]) -> Optional[ProperType]:
        if type_comment is None:
            return None
        else:
            lineno = n.lineno
            extra_ignore, typ = parse_type_comment(type_comment,
                                                   lineno,
                                                   n.col_offset,
                                                   self.errors,
                                                   assume_str_is_unicode=self.unicode_literals)
            if extra_ignore is not None:
                self.type_ignores[lineno] = extra_ignore
            return typ

    op_map = {
        ast27.Add: '+',
        ast27.Sub: '-',
        ast27.Mult: '*',
        ast27.Div: '/',
        ast27.Mod: '%',
        ast27.Pow: '**',
        ast27.LShift: '<<',
        ast27.RShift: '>>',
        ast27.BitOr: '|',
        ast27.BitXor: '^',
        ast27.BitAnd: '&',
        ast27.FloorDiv: '//'
    }  # type: Final[Dict[typing.Type[AST], str]]

    def from_operator(self, op: ast27.operator) -> str:
        op_name = ASTConverter.op_map.get(type(op))
        if op_name is None:
            raise RuntimeError('Unknown operator ' + str(type(op)))
        elif op_name == '@':
            raise RuntimeError('mypy does not support the MatMult operator')
        else:
            return op_name

    comp_op_map = {
        ast27.Gt: '>',
        ast27.Lt: '<',
        ast27.Eq: '==',
        ast27.GtE: '>=',
        ast27.LtE: '<=',
        ast27.NotEq: '!=',
        ast27.Is: 'is',
        ast27.IsNot: 'is not',
        ast27.In: 'in',
        ast27.NotIn: 'not in'
    }  # type: Final[Dict[typing.Type[AST], str]]

    def from_comp_operator(self, op: ast27.cmpop) -> str:
        op_name = ASTConverter.comp_op_map.get(type(op))
        if op_name is None:
            raise RuntimeError('Unknown comparison operator ' + str(type(op)))
        else:
            return op_name

    def as_block(self, stmts: List[ast27.stmt], lineno: int) -> Optional[Block]:
        b = None
        if stmts:
            b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
            b.set_line(lineno)
        return b

    def as_required_block(self, stmts: List[ast27.stmt], lineno: int) -> Block:
        assert stmts  # must be non-empty
        b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts)))
        b.set_line(lineno)
        return b

    def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]:
        ret = []  # type: List[Statement]
        current_overload = []  # type: List[OverloadPart]
        current_overload_name = None  # type: Optional[str]
        for stmt in stmts:
            if (current_overload_name is not None
                    and isinstance(stmt, (Decorator, FuncDef))
                    and stmt.name() == current_overload_name):
                current_overload.append(stmt)
            else:
                if len(current_overload) == 1:
                    ret.append(current_overload[0])
                elif len(current_overload) > 1:
                    ret.append(OverloadedFuncDef(current_overload))

                if isinstance(stmt, Decorator):
                    current_overload = [stmt]
                    current_overload_name = stmt.name()
                else:
                    current_overload = []
                    current_overload_name = None
                    ret.append(stmt)

        if len(current_overload) == 1:
            ret.append(current_overload[0])
        elif len(current_overload) > 1:
            ret.append(OverloadedFuncDef(current_overload))
        return ret

    def in_method_scope(self) -> bool:
        return self.class_and_function_stack[-2:] == ['C', 'F']

    def translate_module_id(self, id: str) -> str:
        """Return the actual, internal module id for a source text id.

        For example, translate '__builtin__' in Python 2 to 'builtins'.
        """
        if id == self.options.custom_typing_module:
            return 'typing'
        elif id == '__builtin__':
            # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation
            #   is named __builtin__.py (there is another layer of translation elsewhere).
            return 'builtins'
        return id

    def visit_Module(self, mod: ast27.Module) -> MypyFile:
        self.type_ignores = {ti.lineno: parse_type_ignore_tag(ti.tag)  # type: ignore[attr-defined]
                             for ti in mod.type_ignores}
        body = self.fix_function_overloads(self.translate_stmt_list(mod.body))
        return MypyFile(body,
                        self.imports,
                        False,
                        self.type_ignores,
                        )

    # --- stmt ---
    # FunctionDef(identifier name, arguments args,
    #             stmt* body, expr*    decorator_list, expr? returns, string? type_comment)
    # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
    #              arg? kwarg, expr* defaults)
    def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
        self.class_and_function_stack.append('F')
        lineno = n.lineno
        converter = TypeConverter(self.errors, line=lineno, override_column=n.col_offset,
                                  assume_str_is_unicode=self.unicode_literals)
        args, decompose_stmts = self.transform_args(n.args, lineno)

        arg_kinds = [arg.kind for arg in args]
        arg_names = [arg.variable.name() for arg in args]  # type: List[Optional[str]]
        arg_names = [None if argument_elide_name(name) else name for name in arg_names]
        if special_function_elide_names(n.name):
            arg_names = [None] * len(arg_names)

        arg_types = []  # type: List[Optional[Type]]
        type_comment = n.type_comment
        if (n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list)):
            arg_types = [None] * len(args)
            return_type = None
        elif type_comment is not None and len(type_comment) > 0:
            try:
                func_type_ast = ast3_parse(type_comment, '<func_type>', 'func_type')
                assert isinstance(func_type_ast, ast3.FunctionType)
                # for ellipsis arg
                if (len(func_type_ast.argtypes) == 1 and
                        isinstance(func_type_ast.argtypes[0], ast3.Ellipsis)):
                    arg_types = [a.type_annotation
                                 if a.type_annotation is not None
                                 else AnyType(TypeOfAny.unannotated)
                                 for a in args]
                else:
                    # PEP 484 disallows both type annotations and type comments
                    if any(a.type_annotation is not None for a in args):
                        self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
                    arg_types = [a if a is not None else AnyType(TypeOfAny.unannotated) for
                                 a in converter.translate_expr_list(func_type_ast.argtypes)]
                return_type = converter.visit(func_type_ast.returns)

                # add implicit self type
                if self.in_method_scope() and len(arg_types) < len(args):
                    arg_types.insert(0, AnyType(TypeOfAny.special_form))
            except SyntaxError:
                stripped_type = type_comment.split("#", 2)[0].strip()
                err_msg = "{} '{}'".format(TYPE_COMMENT_SYNTAX_ERROR, stripped_type)
                self.fail(err_msg, lineno, n.col_offset)
                arg_types = [AnyType(TypeOfAny.from_error)] * len(args)
                return_type = AnyType(TypeOfAny.from_error)
        else:
            arg_types = [a.type_annotation for a in args]
            return_type = converter.visit(None)

        for arg, arg_type in zip(args, arg_types):
            self.set_type_optional(arg_type, arg.initializer)

        func_type = None
        if any(arg_types) or return_type:
            if len(arg_types) != 1 and any(isinstance(t, EllipsisType)
                                           for t in arg_types):
                self.fail("Ellipses cannot accompany other argument types "
                          "in function type signature", lineno, n.col_offset)
            elif len(arg_types) > len(arg_kinds):
                self.fail('Type signature has too many arguments', lineno, n.col_offset,
                          blocker=False)
            elif len(arg_types) < len(arg_kinds):
                self.fail('Type signature has too few arguments', lineno, n.col_offset,
                          blocker=False)
            else:
                any_type = AnyType(TypeOfAny.unannotated)
                func_type = CallableType([a if a is not None else any_type for a in arg_types],
                                        arg_kinds,
                                        arg_names,
                                        return_type if return_type is not None else any_type,
                                        _dummy_fallback)

        body = self.as_required_block(n.body, lineno)
        if decompose_stmts:
            body.body = decompose_stmts + body.body
        func_def = FuncDef(n.name,
                       args,
                       body,
                       func_type)
        if isinstance(func_def.type, CallableType):
            # semanal.py does some in-place modifications we want to avoid
            func_def.unanalyzed_type = func_def.type.copy_modified()
        if func_type is not None:
            func_type.definition = func_def
            func_type.line = lineno

        if n.decorator_list:
            var = Var(func_def.name())
            var.is_ready = False
            var.set_line(n.decorator_list[0].lineno)

            func_def.is_decorated = True
            func_def.set_line(lineno + len(n.decorator_list))
            func_def.body.set_line(func_def.get_line())
            dec = Decorator(func_def, self.translate_expr_list(n.decorator_list), var)
            dec.set_line(lineno, n.col_offset)
            retval = dec  # type: Statement
        else:
            # Overrides set_line -- can't use self.set_line
            func_def.set_line(lineno, n.col_offset)
            retval = func_def
        self.class_and_function_stack.pop()
        return retval

    def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None:
        if self.options.no_implicit_optional:
            return
        # Indicate that type should be wrapped in an Optional if arg is initialized to None.
        optional = isinstance(initializer, NameExpr) and initializer.name == 'None'
        if isinstance(type, UnboundType):
            type.optional = optional

    def transform_args(self,
                       n: ast27.arguments,
                       line: int,
                       ) -> Tuple[List[Argument], List[Statement]]:
        type_comments = n.type_comments  # type: Sequence[Optional[str]]
        converter = TypeConverter(self.errors, line=line,
                                  assume_str_is_unicode=self.unicode_literals)
        decompose_stmts = []  # type: List[Statement]

        n_args = n.args
        args = [(self.convert_arg(i, arg, line, decompose_stmts),
                 self.get_type(i, type_comments, converter))
                for i, arg in enumerate(n_args)]
        defaults = self.translate_expr_list(n.defaults)
        names = [name for arg in n_args for name in self.extract_names(arg)]  # type: List[str]

        new_args = []  # type: List[Argument]
        num_no_defaults = len(args) - len(defaults)
        # positional arguments without defaults
        for a, annotation in args[:num_no_defaults]:
            new_args.append(Argument(a, annotation, None, ARG_POS))

        # positional arguments with defaults
        for (a, annotation), d in zip(args[num_no_defaults:], defaults):
            new_args.append(Argument(a, annotation, d, ARG_OPT))

        # *arg
        if n.vararg is not None:
            new_args.append(Argument(Var(n.vararg),
                                     self.get_type(len(args), type_comments, converter),
                                     None,
                                     ARG_STAR))
            names.append(n.vararg)

        # **kwarg
        if n.kwarg is not None:
            typ = self.get_type(len(args) + (0 if n.vararg is None else 1),
                                type_comments,
                                converter)
            new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2))
            names.append(n.kwarg)

        # We don't have any context object to give, but we have closed around the line num
        def fail_arg(msg: str, arg: None) -> None:
            self.fail(msg, line, 0)
        check_arg_names(names, [None] * len(names), fail_arg)

        return new_args, decompose_stmts

    def extract_names(self, arg: ast27.expr) -> List[str]:
        if isinstance(arg, Name):
            return [arg.id]
        elif isinstance(arg, ast27_Tuple):
            return [name for elt in arg.elts for name in self.extract_names(elt)]
        else:
            return []

    def convert_arg(self, index: int, arg: ast27.expr, line: int,
                    decompose_stmts: List[Statement]) -> Var:
        if isinstance(arg, Name):
            v = arg.id
        elif isinstance(arg, ast27_Tuple):
            v = '__tuple_arg_{}'.format(index + 1)
            rvalue = NameExpr(v)
            rvalue.set_line(line)
            assignment = AssignmentStmt([self.visit(arg)], rvalue)
            assignment.set_line(line)
            decompose_stmts.append(assignment)
        else:
            raise RuntimeError("'{}' is not a valid argument.".format(ast27.dump(arg)))
        return Var(v)

    def get_type(self,
                 i: int,
                 type_comments: Sequence[Optional[str]],
                 converter: TypeConverter) -> Optional[Type]:
        if i < len(type_comments):
            comment = type_comments[i]
            if comment is not None:
                typ = converter.visit_raw_str(comment)
                extra_ignore = TYPE_IGNORE_PATTERN.match(comment)
                if extra_ignore:
                    tag = cast(Any, extra_ignore).group(1)  # type: Optional[str]
                    ignored = parse_type_ignore_tag(tag)
                    self.type_ignores[converter.line] = ignored
                return typ
        return None

    def stringify_name(self, n: AST) -> str:
        if isinstance(n, Name):
            return n.id
        elif isinstance(n, Attribute):
            return "{}.{}".format(self.stringify_name(n.value), n.attr)
        else:
            assert False, "can't stringify " + str(type(n))

    # ClassDef(identifier name,
    #  expr* bases,
    #  keyword* keywords,
    #  stmt* body,
    #  expr* decorator_list)
    def visit_ClassDef(self, n: ast27.ClassDef) -> ClassDef:
        self.class_and_function_stack.append('C')

        cdef = ClassDef(n.name,
                        self.as_required_block(n.body, n.lineno),
                        None,
                        self.translate_expr_list(n.bases),
                        metaclass=None)
        cdef.decorators = self.translate_expr_list(n.decorator_list)
        cdef.line = n.lineno + len(n.decorator_list)
        cdef.column = n.col_offset
        cdef.end_line = n.lineno
        self.class_and_function_stack.pop()
        return cdef

    # Return(expr? value)
    def visit_Return(self, n: ast27.Return) -> ReturnStmt:
        stmt = ReturnStmt(self.visit(n.value))
        return self.set_line(stmt, n)

    # Delete(expr* targets)
    def visit_Delete(self, n: ast27.Delete) -> DelStmt:
        if len(n.targets) > 1:
            tup = TupleExpr(self.translate_expr_list(n.targets))
            tup.set_line(n.lineno)
            stmt = DelStmt(tup)
        else:
            stmt = DelStmt(self.visit(n.targets[0]))
        return self.set_line(stmt, n)

    # Assign(expr* targets, expr value, string? type_comment)
    def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt:
        typ = self.translate_type_comment(n, n.type_comment)
        stmt = AssignmentStmt(self.translate_expr_list(n.targets),
                              self.visit(n.value),
                              type=typ)
        return self.set_line(stmt, n)

    # AugAssign(expr target, operator op, expr value)
    def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt:
        stmt = OperatorAssignmentStmt(self.from_operator(n.op),
                                      self.visit(n.target),
                                      self.visit(n.value))
        return self.set_line(stmt, n)

    # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
    def visit_For(self, n: ast27.For) -> ForStmt:
        typ = self.translate_type_comment(n, n.type_comment)
        stmt = ForStmt(self.visit(n.target),
                       self.visit(n.iter),
                       self.as_required_block(n.body, n.lineno),
                       self.as_block(n.orelse, n.lineno),
                       typ)
        return self.set_line(stmt, n)

    # While(expr test, stmt* body, stmt* orelse)
    def visit_While(self, n: ast27.While) -> WhileStmt:
        stmt = WhileStmt(self.visit(n.test),
                         self.as_required_block(n.body, n.lineno),
                         self.as_block(n.orelse, n.lineno))
        return self.set_line(stmt, n)

    # If(expr test, stmt* body, stmt* orelse)
    def visit_If(self, n: ast27.If) -> IfStmt:
        stmt = IfStmt([self.visit(n.test)],
                      [self.as_required_block(n.body, n.lineno)],
                      self.as_block(n.orelse, n.lineno))
        return self.set_line(stmt, n)

    # With(withitem* items, stmt* body, string? type_comment)
    def visit_With(self, n: ast27.With) -> WithStmt:
        typ = self.translate_type_comment(n, n.type_comment)
        stmt = WithStmt([self.visit(n.context_expr)],
                        [self.visit(n.optional_vars)],
                        self.as_required_block(n.body, n.lineno),
                        typ)
        return self.set_line(stmt, n)

    def visit_Raise(self, n: ast27.Raise) -> RaiseStmt:
        if n.type is None:
            e = None
        else:
            if n.inst is None:
                e = self.visit(n.type)
            else:
                if n.tback is None:
                    e = TupleExpr([self.visit(n.type), self.visit(n.inst)])
                else:
                    e = TupleExpr([self.visit(n.type), self.visit(n.inst), self.visit(n.tback)])

        stmt = RaiseStmt(e, None)
        return self.set_line(stmt, n)

    # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
    def visit_TryExcept(self, n: ast27.TryExcept) -> TryStmt:
        stmt = self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno)
        return self.set_line(stmt, n)

    def visit_TryFinally(self, n: ast27.TryFinally) -> TryStmt:
        if len(n.body) == 1 and isinstance(n.body[0], ast27.TryExcept):
            stmt = self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno)
        else:
            stmt = self.try_handler(n.body, [], [], n.finalbody, n.lineno)
        return self.set_line(stmt, n)

    def try_handler(self,
                    body: List[ast27.stmt],
                    handlers: List[ast27.ExceptHandler],
                    orelse: List[ast27.stmt],
                    finalbody: List[ast27.stmt],
                    lineno: int) -> TryStmt:
        vs = []  # type: List[Optional[NameExpr]]
        for item in handlers:
            if item.name is None:
                vs.append(None)
            elif isinstance(item.name, Name):
                vs.append(NameExpr(item.name.id))
            else:
                self.fail("Sorry, `except <expr>, <anything but a name>` is not supported",
                          item.lineno, item.col_offset)
                vs.append(None)
        types = [self.visit(h.type) for h in handlers]
        handlers_ = [self.as_required_block(h.body, h.lineno) for h in handlers]

        return TryStmt(self.as_required_block(body, lineno),
                       vs,
                       types,
                       handlers_,
                       self.as_block(orelse, lineno),
                       self.as_block(finalbody, lineno))

    def visit_Print(self, n: ast27.Print) -> PrintStmt:
        stmt = PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest))
        return self.set_line(stmt, n)

    def visit_Exec(self, n: ast27.Exec) -> ExecStmt:
        stmt = ExecStmt(self.visit(n.body),
                     self.visit(n.globals),
                     self.visit(n.locals))
        return self.set_line(stmt, n)

    def visit_Repr(self, n: ast27.Repr) -> BackquoteExpr:
        stmt = BackquoteExpr(self.visit(n.value))
        return self.set_line(stmt, n)

    # Assert(expr test, expr? msg)
    def visit_Assert(self, n: ast27.Assert) -> AssertStmt:
        stmt = AssertStmt(self.visit(n.test), self.visit(n.msg))
        return self.set_line(stmt, n)

    # Import(alias* names)
    def visit_Import(self, n: ast27.Import) -> Import:
        names = []  # type: List[Tuple[str, Optional[str]]]
        for alias in n.names:
            name = self.translate_module_id(alias.name)
            asname = alias.asname
            if asname is None and name != alias.name:
                # if the module name has been translated (and it's not already
                # an explicit import-as), make it an implicit import-as the
                # original name
                asname = alias.name
            names.append((name, asname))
        i = Import(names)
        self.imports.append(i)
        return self.set_line(i, n)

    # ImportFrom(identifier? module, alias* names, int? level)
    def visit_ImportFrom(self, n: ast27.ImportFrom) -> ImportBase:
        assert n.level is not None
        if len(n.names) == 1 and n.names[0].name == '*':
            mod = n.module if n.module is not None else ''
            i = ImportAll(mod, n.level)  # type: ImportBase
        else:
            module_id = self.translate_module_id(n.module) if n.module is not None else ''
            i = ImportFrom(module_id, n.level, [(a.name, a.asname) for a in n.names])

            # See comments in the constructor for more information about this field.
            if module_id == '__future__' and any(a.name == 'unicode_literals' for a in n.names):
                self.unicode_literals = True
        self.imports.append(i)
        return self.set_line(i, n)

    # Global(identifier* names)
    def visit_Global(self, n: ast27.Global) -> GlobalDecl:
        stmt = GlobalDecl(n.names)
        return self.set_line(stmt, n)

    # Expr(expr value)
    def visit_Expr(self, n: ast27.Expr) -> ExpressionStmt:
        value = self.visit(n.value)
        stmt = ExpressionStmt(value)
        return self.set_line(stmt, n)

    # Pass
    def visit_Pass(self, n: ast27.Pass) -> PassStmt:
        stmt = PassStmt()
        return self.set_line(stmt, n)

    # Break
    def visit_Break(self, n: ast27.Break) -> BreakStmt:
        stmt = BreakStmt()
        return self.set_line(stmt, n)

    # Continue
    def visit_Continue(self, n: ast27.Continue) -> ContinueStmt:
        stmt = ContinueStmt()
        return self.set_line(stmt, n)

    # --- expr ---

    # BoolOp(boolop op, expr* values)
    def visit_BoolOp(self, n: ast27.BoolOp) -> OpExpr:
        # mypy translates (1 and 2 and 3) as (1 and (2 and 3))
        assert len(n.values) >= 2
        if isinstance(n.op, ast27.And):
            op = 'and'
        elif isinstance(n.op, ast27.Or):
            op = 'or'
        else:
            raise RuntimeError('unknown BoolOp ' + str(type(n)))

        # potentially inefficient!
        e = self.group(self.translate_expr_list(n.values), op)
        return self.set_line(e, n)

    def group(self, vals: List[Expression], op: str) -> OpExpr:
        if len(vals) == 2:
            return OpExpr(op, vals[0], vals[1])
        else:
            return OpExpr(op, vals[0], self.group(vals[1:], op))

    # BinOp(expr left, operator op, expr right)
    def visit_BinOp(self, n: ast27.BinOp) -> OpExpr:
        op = self.from_operator(n.op)

        if op is None:
            raise RuntimeError('cannot translate BinOp ' + str(type(n.op)))

        e = OpExpr(op, self.visit(n.left), self.visit(n.right))
        return self.set_line(e, n)

    # UnaryOp(unaryop op, expr operand)
    def visit_UnaryOp(self, n: ast27.UnaryOp) -> UnaryExpr:
        op = None
        if isinstance(n.op, ast27.Invert):
            op = '~'
        elif isinstance(n.op, ast27.Not):
            op = 'not'
        elif isinstance(n.op, ast27.UAdd):
            op = '+'
        elif isinstance(n.op, ast27.USub):
            op = '-'

        if op is None:
            raise RuntimeError('cannot translate UnaryOp ' + str(type(n.op)))

        e = UnaryExpr(op, self.visit(n.operand))
        return self.set_line(e, n)

    # Lambda(arguments args, expr body)
    def visit_Lambda(self, n: ast27.Lambda) -> LambdaExpr:
        args, decompose_stmts = self.transform_args(n.args, n.lineno)

        n_body = ast27.Return(n.body)
        n_body.lineno = n.body.lineno
        n_body.col_offset = n.body.col_offset
        body = self.as_required_block([n_body], n.lineno)
        if decompose_stmts:
            body.body = decompose_stmts + body.body

        e = LambdaExpr(args, body)
        e.set_line(n.lineno, n.col_offset)  # Overrides set_line -- can't use self.set_line
        return e

    # IfExp(expr test, expr body, expr orelse)
    def visit_IfExp(self, n: ast27.IfExp) -> ConditionalExpr:
        e = ConditionalExpr(self.visit(n.test),
                            self.visit(n.body),
                            self.visit(n.orelse))
        return self.set_line(e, n)

    # Dict(expr* keys, expr* values)
    def visit_Dict(self, n: ast27.Dict) -> DictExpr:
        e = DictExpr(list(zip(self.translate_expr_list(n.keys),
                              self.translate_expr_list(n.values))))
        return self.set_line(e, n)

    # Set(expr* elts)
    def visit_Set(self, n: ast27.Set) -> SetExpr:
        e = SetExpr(self.translate_expr_list(n.elts))
        return self.set_line(e, n)

    # ListComp(expr elt, comprehension* generators)
    def visit_ListComp(self, n: ast27.ListComp) -> ListComprehension:
        e = ListComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n)))
        return self.set_line(e, n)

    # SetComp(expr elt, comprehension* generators)
    def visit_SetComp(self, n: ast27.SetComp) -> SetComprehension:
        e = SetComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n)))
        return self.set_line(e, n)

    # DictComp(expr key, expr value, comprehension* generators)
    def visit_DictComp(self, n: ast27.DictComp) -> DictionaryComprehension:
        targets = [self.visit(c.target) for c in n.generators]
        iters = [self.visit(c.iter) for c in n.generators]
        ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
        e = DictionaryComprehension(self.visit(n.key),
                                    self.visit(n.value),
                                    targets,
                                    iters,
                                    ifs_list,
                                    [False for _ in n.generators])
        return self.set_line(e, n)

    # GeneratorExp(expr elt, comprehension* generators)
    def visit_GeneratorExp(self, n: ast27.GeneratorExp) -> GeneratorExpr:
        targets = [self.visit(c.target) for c in n.generators]
        iters = [self.visit(c.iter) for c in n.generators]
        ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators]
        e = GeneratorExpr(self.visit(n.elt),
                          targets,
                          iters,
                          ifs_list,
                          [False for _ in n.generators])
        return self.set_line(e, n)

    # Yield(expr? value)
    def visit_Yield(self, n: ast27.Yield) -> YieldExpr:
        e = YieldExpr(self.visit(n.value))
        return self.set_line(e, n)

    # Compare(expr left, cmpop* ops, expr* comparators)
    def visit_Compare(self, n: ast27.Compare) -> ComparisonExpr:
        operators = [self.from_comp_operator(o) for o in n.ops]
        operands = self.translate_expr_list([n.left] + n.comparators)
        e = ComparisonExpr(operators, operands)
        return self.set_line(e, n)

    # Call(expr func, expr* args, keyword* keywords)
    # keyword = (identifier? arg, expr value)
    def visit_Call(self, n: Call) -> CallExpr:
        arg_types = []  # type: List[ast27.expr]
        arg_kinds = []  # type: List[int]
        signature = []  # type: List[Optional[str]]

        args = n.args
        arg_types.extend(args)
        arg_kinds.extend(ARG_POS for a in args)
        signature.extend(None for a in args)

        if n.starargs is not None:
            arg_types.append(n.starargs)
            arg_kinds.append(ARG_STAR)
            signature.append(None)

        keywords = n.keywords
        arg_types.extend(k.value for k in keywords)
        arg_kinds.extend(ARG_NAMED for k in keywords)
        signature.extend(k.arg for k in keywords)

        if n.kwargs is not None:
            arg_types.append(n.kwargs)
            arg_kinds.append(ARG_STAR2)
            signature.append(None)

        e = CallExpr(self.visit(n.func),
                     self.translate_expr_list(arg_types),
                     arg_kinds,
                     signature)
        return self.set_line(e, n)

    # Num(object n) -- a number as a PyObject.
    def visit_Num(self, n: ast27.Num) -> Expression:
        # The n field has the type complex, but complex isn't *really*
        # a parent of int and float, and this causes isinstance below
        # to think that the complex branch is always picked. Avoid
        # this by throwing away the type.
        value = n.n  # type: object
        is_inverse = False
        if str(n.n).startswith('-'):  # Hackish because of complex.
            value = -n.n
            is_inverse = True

        if isinstance(value, int):
            expr = IntExpr(value)  # type: Expression
        elif isinstance(value, float):
            expr = FloatExpr(value)
        elif isinstance(value, complex):
            expr = ComplexExpr(value)
        else:
            raise RuntimeError('num not implemented for ' + str(type(n.n)))

        if is_inverse:
            expr = UnaryExpr('-', expr)

        return self.set_line(expr, n)

    # Str(string s)
    def visit_Str(self, n: ast27.Str) -> Expression:
        # Note: typed_ast.ast27 will handled unicode_literals for us. If
        # n.s is of type 'bytes', we know unicode_literals was not enabled;
        # otherwise we know it was.
        #
        # Note that the following code is NOT run when parsing Python 2.7 stubs:
        # we always parse stub files (no matter what version) using the Python 3
        # parser. This is also why string literals in Python 2.7 stubs are assumed
        # to be unicode.
        if isinstance(n.s, bytes):
            contents = bytes_to_human_readable_repr(n.s)
            e = StrExpr(contents, from_python_3=False)  # type: Union[StrExpr, UnicodeExpr]
            return self.set_line(e, n)
        else:
            e = UnicodeExpr(n.s)
            return self.set_line(e, n)

    # Ellipsis
    def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr:
        return EllipsisExpr()

    # Attribute(expr value, identifier attr, expr_context ctx)
    def visit_Attribute(self, n: Attribute) -> Expression:
        # First create MemberExpr and then potentially replace with a SuperExpr
        # to improve performance when compiled. The check for "super()" will be
        # faster with native AST nodes. Note also that super expressions are
        # less common than normal member expressions.
        member_expr = MemberExpr(self.visit(n.value), n.attr)
        obj = member_expr.expr
        if (isinstance(obj, CallExpr) and
                isinstance(obj.callee, NameExpr) and
                obj.callee.name == 'super'):
            e = SuperExpr(member_expr.name, obj)  # type: Expression
        else:
            e = member_expr
        return self.set_line(e, n)

    # Subscript(expr value, slice slice, expr_context ctx)
    def visit_Subscript(self, n: ast27.Subscript) -> IndexExpr:
        e = IndexExpr(self.visit(n.value), self.visit(n.slice))
        self.set_line(e, n)
        if isinstance(e.index, SliceExpr):
            # Slice has no line/column in the raw ast.
            e.index.line = e.line
            e.index.column = e.column
        return e

    # Name(identifier id, expr_context ctx)
    def visit_Name(self, n: Name) -> NameExpr:
        e = NameExpr(n.id)
        return self.set_line(e, n)

    # List(expr* elts, expr_context ctx)
    def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]:
        expr_list = [self.visit(e) for e in n.elts]  # type: List[Expression]
        if isinstance(n.ctx, ast27.Store):
            # [x, y] = z and (x, y) = z means exactly the same thing
            e = TupleExpr(expr_list)  # type: Union[ListExpr, TupleExpr]
        else:
            e = ListExpr(expr_list)
        return self.set_line(e, n)

    # Tuple(expr* elts, expr_context ctx)
    def visit_Tuple(self, n: ast27_Tuple) -> TupleExpr:
        e = TupleExpr([self.visit(e) for e in n.elts])
        return self.set_line(e, n)

    # --- slice ---

    # Slice(expr? lower, expr? upper, expr? step)
    def visit_Slice(self, n: ast27.Slice) -> SliceExpr:
        return SliceExpr(self.visit(n.lower),
                         self.visit(n.upper),
                         self.visit(n.step))

    # ExtSlice(slice* dims)
    def visit_ExtSlice(self, n: ast27.ExtSlice) -> TupleExpr:
        return TupleExpr(self.translate_expr_list(n.dims))

    # Index(expr value)
    def visit_Index(self, n: ast27.Index) -> Expression:
        return self.visit(n.value)