pep612: more semantic analysis for paramspec (#9422)

Linking #8645

Like #9339, much of this is in #9250. However, this PR is a little less self 
contained, in that it causes several future TODOs.

A lot of the changes here are necessitated by changing the type of 
CallableType.variables to Sequence[TypeVarLikeDef]. This is nice, because 
it informs me where I need to make changes in the future / integrates a little 
bit better with existing type var stuff.

mypy/applytype.py

@@ -4,11 +4,55 @@
 import mypy.sametypes
 from mypy.expandtype import expand_type
 from mypy.types import (
-    Type, TypeVarId, TypeVarType, CallableType, AnyType, PartialType, get_proper_types
+    Type, TypeVarId, TypeVarType, CallableType, AnyType, PartialType, get_proper_types,
+    TypeVarDef, TypeVarLikeDef, ProperType
 )
 from mypy.nodes import Context
 
 
+def get_target_type(
+    tvar: TypeVarLikeDef,
+    type: ProperType,
+    callable: CallableType,
+    report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
+    context: Context,
+    skip_unsatisfied: bool
+) -> Optional[Type]:
+    # TODO(shantanu): fix for ParamSpecDef
+    assert isinstance(tvar, TypeVarDef)
+    values = get_proper_types(tvar.values)
+    if values:
+        if isinstance(type, AnyType):
+            return type
+        if isinstance(type, TypeVarType) and type.values:
+            # Allow substituting T1 for T if every allowed value of T1
+            # is also a legal value of T.
+            if all(any(mypy.sametypes.is_same_type(v, v1) for v in values)
+                   for v1 in type.values):
+                return type
+        matching = []
+        for value in values:
+            if mypy.subtypes.is_subtype(type, value):
+                matching.append(value)
+        if matching:
+            best = matching[0]
+            # If there are more than one matching value, we select the narrowest
+            for match in matching[1:]:
+                if mypy.subtypes.is_subtype(match, best):
+                    best = match
+            return best
+        if skip_unsatisfied:
+            return None
+        report_incompatible_typevar_value(callable, type, tvar.name, context)
+    else:
+        upper_bound = tvar.upper_bound
+        if not mypy.subtypes.is_subtype(type, upper_bound):
+            if skip_unsatisfied:
+                return None
+            report_incompatible_typevar_value(callable, type, tvar.name, context)
+    return type
+
+
 def apply_generic_arguments(
         callable: CallableType, orig_types: Sequence[Optional[Type]],
         report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
@@ -29,52 +73,20 @@ def apply_generic_arguments(
     # Check that inferred type variable values are compatible with allowed
     # values and bounds.  Also, promote subtype values to allowed values.
     types = get_proper_types(orig_types)
-    for i, type in enumerate(types):
+
+    # Create a map from type variable id to target type.
+    id_to_type = {}  # type: Dict[TypeVarId, Type]
+
+    for tvar, type in zip(tvars, types):
         assert not isinstance(type, PartialType), "Internal error: must never apply partial type"
-        values = get_proper_types(callable.variables[i].values)
         if type is None:
             continue
-        if values:
-            if isinstance(type, AnyType):
-                continue
-            if isinstance(type, TypeVarType) and type.values:
-                # Allow substituting T1 for T if every allowed value of T1
-                # is also a legal value of T.
-                if all(any(mypy.sametypes.is_same_type(v, v1) for v in values)
-                       for v1 in type.values):
-                    continue
-            matching = []
-            for value in values:
-                if mypy.subtypes.is_subtype(type, value):
-                    matching.append(value)
-            if matching:
-                best = matching[0]
-                # If there are more than one matching value, we select the narrowest
-                for match in matching[1:]:
-                    if mypy.subtypes.is_subtype(match, best):
-                        best = match
-                types[i] = best
-            else:
-                if skip_unsatisfied:
-                    types[i] = None
-                else:
-                    report_incompatible_typevar_value(callable, type, callable.variables[i].name,
-                                                      context)
-        else:
-            upper_bound = callable.variables[i].upper_bound
-            if not mypy.subtypes.is_subtype(type, upper_bound):
-                if skip_unsatisfied:
-                    types[i] = None
-                else:
-                    report_incompatible_typevar_value(callable, type, callable.variables[i].name,
-                                                      context)
 
-    # Create a map from type variable id to target type.
-    id_to_type = {}  # type: Dict[TypeVarId, Type]
-    for i, tv in enumerate(tvars):
-        typ = types[i]
-        if typ:
-            id_to_type[tv.id] = typ
+        target_type = get_target_type(
+            tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied
+        )
+        if target_type is not None:
+            id_to_type[tvar.id] = target_type
 
     # Apply arguments to argument types.
     arg_types = [expand_type(at, id_to_type) for at in callable.arg_types]

mypy/checker.py

@@ -1389,15 +1389,14 @@ def expand_typevars(self, defn: FuncItem,
                         typ: CallableType) -> List[Tuple[FuncItem, CallableType]]:
         # TODO use generator
         subst = []  # type: List[List[Tuple[TypeVarId, Type]]]
-        tvars = typ.variables or []
-        tvars = tvars[:]
+        tvars = list(typ.variables) or []
         if defn.info:
             # Class type variables
             tvars += defn.info.defn.type_vars or []
+        # TODO(shantanu): audit for paramspec
         for tvar in tvars:
-            if tvar.values:
-                subst.append([(tvar.id, value)
-                              for value in tvar.values])
+            if isinstance(tvar, TypeVarDef) and tvar.values:
+                subst.append([(tvar.id, value) for value in tvar.values])
         # Make a copy of the function to check for each combination of
         # value restricted type variables. (Except when running mypyc,
         # where we need one canonical version of the function.)

mypy/checkexpr.py

@@ -4311,6 +4311,8 @@ def merge_typevars_in_callables_by_name(
             for tvdef in target.variables:
                 name = tvdef.fullname
                 if name not in unique_typevars:
+                    # TODO(shantanu): fix for ParamSpecDef
+                    assert isinstance(tvdef, TypeVarDef)
                     unique_typevars[name] = TypeVarType(tvdef)
                     variables.append(tvdef)
                 rename[tvdef.id] = unique_typevars[name]

mypy/checkmember.py

@@ -1,11 +1,11 @@
 """Type checking of attribute access"""
 
-from typing import cast, Callable, Optional, Union, List
+from typing import cast, Callable, Optional, Union, Sequence
 from typing_extensions import TYPE_CHECKING
 
 from mypy.types import (
-    Type, Instance, AnyType, TupleType, TypedDictType, CallableType, FunctionLike, TypeVarDef,
-    Overloaded, TypeVarType, UnionType, PartialType, TypeOfAny, LiteralType,
+    Type, Instance, AnyType, TupleType, TypedDictType, CallableType, FunctionLike,
+    TypeVarLikeDef, Overloaded, TypeVarType, UnionType, PartialType, TypeOfAny, LiteralType,
     DeletedType, NoneType, TypeType, has_type_vars, get_proper_type, ProperType
 )
 from mypy.nodes import (
@@ -676,7 +676,7 @@ def analyze_class_attribute_access(itype: Instance,
                                    name: str,
                                    mx: MemberContext,
                                    override_info: Optional[TypeInfo] = None,
-                                   original_vars: Optional[List[TypeVarDef]] = None
+                                   original_vars: Optional[Sequence[TypeVarLikeDef]] = None
                                    ) -> Optional[Type]:
     """Analyze access to an attribute on a class object.
 
@@ -839,7 +839,7 @@ def analyze_enum_class_attribute_access(itype: Instance,
 def add_class_tvars(t: ProperType, isuper: Optional[Instance],
                     is_classmethod: bool,
                     original_type: Type,
-                    original_vars: Optional[List[TypeVarDef]] = None) -> Type:
+                    original_vars: Optional[Sequence[TypeVarLikeDef]] = None) -> Type:
     """Instantiate type variables during analyze_class_attribute_access,
     e.g T and Q in the following:
 
@@ -883,7 +883,7 @@ class B(A[str]): pass
             assert isuper is not None
             t = cast(CallableType, expand_type_by_instance(t, isuper))
             freeze_type_vars(t)
-        return t.copy_modified(variables=tvars + t.variables)
+        return t.copy_modified(variables=list(tvars) + list(t.variables))
     elif isinstance(t, Overloaded):
         return Overloaded([cast(CallableType, add_class_tvars(item, isuper,
                                                               is_classmethod, original_type,

mypy/expandtype.py

@@ -40,6 +40,8 @@ def freshen_function_type_vars(callee: F) -> F:
         tvdefs = []
         tvmap = {}  # type: Dict[TypeVarId, Type]
         for v in callee.variables:
+            # TODO(shantanu): fix for ParamSpecDef
+            assert isinstance(v, TypeVarDef)
             tvdef = TypeVarDef.new_unification_variable(v)
             tvdefs.append(tvdef)
             tvmap[v.id] = TypeVarType(tvdef)

mypy/fixup.py

@@ -11,7 +11,8 @@
 from mypy.types import (
     CallableType, Instance, Overloaded, TupleType, TypedDictType,
     TypeVarType, UnboundType, UnionType, TypeVisitor, LiteralType,
-    TypeType, NOT_READY, TypeAliasType, AnyType, TypeOfAny)
+    TypeType, NOT_READY, TypeAliasType, AnyType, TypeOfAny, TypeVarDef
+)
 from mypy.visitor import NodeVisitor
 from mypy.lookup import lookup_fully_qualified
 
@@ -183,10 +184,11 @@ def visit_callable_type(self, ct: CallableType) -> None:
         if ct.ret_type is not None:
             ct.ret_type.accept(self)
         for v in ct.variables:
-            if v.values:
-                for val in v.values:
-                    val.accept(self)
-            v.upper_bound.accept(self)
+            if isinstance(v, TypeVarDef):
+                if v.values:
+                    for val in v.values:
+                        val.accept(self)
+                v.upper_bound.accept(self)
         for arg in ct.bound_args:
             if arg:
                 arg.accept(self)

mypy/messages.py

@@ -21,7 +21,7 @@
 from mypy.errors import Errors
 from mypy.types import (
     Type, CallableType, Instance, TypeVarType, TupleType, TypedDictType, LiteralType,
-    UnionType, NoneType, AnyType, Overloaded, FunctionLike, DeletedType, TypeType,
+    UnionType, NoneType, AnyType, Overloaded, FunctionLike, DeletedType, TypeType, TypeVarDef,
     UninhabitedType, TypeOfAny, UnboundType, PartialType, get_proper_type, ProperType,
     get_proper_types
 )
@@ -1862,16 +1862,20 @@ def [T <: int] f(self, x: int, y: T) -> None
     if tp.variables:
         tvars = []
         for tvar in tp.variables:
-            upper_bound = get_proper_type(tvar.upper_bound)
-            if (isinstance(upper_bound, Instance) and
-                    upper_bound.type.fullname != 'builtins.object'):
-                tvars.append('{} <: {}'.format(tvar.name, format_type_bare(upper_bound)))
-            elif tvar.values:
-                tvars.append('{} in ({})'
-                             .format(tvar.name, ', '.join([format_type_bare(tp)
-                                                           for tp in tvar.values])))
+            if isinstance(tvar, TypeVarDef):
+                upper_bound = get_proper_type(tvar.upper_bound)
+                if (isinstance(upper_bound, Instance) and
+                        upper_bound.type.fullname != 'builtins.object'):
+                    tvars.append('{} <: {}'.format(tvar.name, format_type_bare(upper_bound)))
+                elif tvar.values:
+                    tvars.append('{} in ({})'
+                                 .format(tvar.name, ', '.join([format_type_bare(tp)
+                                                               for tp in tvar.values])))
+                else:
+                    tvars.append(tvar.name)
             else:
-                tvars.append(tvar.name)
+                # For other TypeVarLikeDefs, just use the repr
+                tvars.append(repr(tvar))
         s = '[{}] {}'.format(', '.join(tvars), s)
     return 'def {}'.format(s)
 

mypy/nodes.py

@@ -2079,7 +2079,7 @@ class TypeVarExpr(TypeVarLikeExpr):
 
     This is also used to represent type variables in symbol tables.
 
-    A type variable is not valid as a type unless bound in a TypeVarScope.
+    A type variable is not valid as a type unless bound in a TypeVarLikeScope.
     That happens within:
 
      1. a generic class that uses the type variable as a type argument or

mypy/plugin.py

@@ -126,7 +126,7 @@ class C: pass
 from mypy.nodes import (
     Expression, Context, ClassDef, SymbolTableNode, MypyFile, CallExpr
 )
-from mypy.tvar_scope import TypeVarScope
+from mypy.tvar_scope import TypeVarLikeScope
 from mypy.types import Type, Instance, CallableType, TypeList, UnboundType, ProperType
 from mypy.messages import MessageBuilder
 from mypy.options import Options
@@ -265,7 +265,7 @@ def fail(self, msg: str, ctx: Context, serious: bool = False, *,
 
     @abstractmethod
     def anal_type(self, t: Type, *,
-                  tvar_scope: Optional[TypeVarScope] = None,
+                  tvar_scope: Optional[TypeVarLikeScope] = None,
                   allow_tuple_literal: bool = False,
                   allow_unbound_tvars: bool = False,
                   report_invalid_types: bool = True,

mypy/plugins/default.py

@@ -10,7 +10,7 @@
 from mypy.plugins.common import try_getting_str_literals
 from mypy.types import (
     Type, Instance, AnyType, TypeOfAny, CallableType, NoneType, TypedDictType,
-    TypeVarType, TPDICT_FB_NAMES, get_proper_type, LiteralType
+    TypeVarDef, TypeVarType, TPDICT_FB_NAMES, get_proper_type, LiteralType
 )
 from mypy.subtypes import is_subtype
 from mypy.typeops import make_simplified_union
@@ -204,6 +204,7 @@ def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType:
             # Tweak the signature to include the value type as context. It's
             # only needed for type inference since there's a union with a type
             # variable that accepts everything.
+            assert isinstance(signature.variables[0], TypeVarDef)
             tv = TypeVarType(signature.variables[0])
             return signature.copy_modified(
                 arg_types=[signature.arg_types[0],
@@ -269,6 +270,7 @@ def typed_dict_pop_signature_callback(ctx: MethodSigContext) -> CallableType:
             # Tweak the signature to include the value type as context. It's
             # only needed for type inference since there's a union with a type
             # variable that accepts everything.
+            assert isinstance(signature.variables[0], TypeVarDef)
             tv = TypeVarType(signature.variables[0])
             typ = make_simplified_union([value_type, tv])
             return signature.copy_modified(