Fixed bugs in two of the conformance tests: "generics_basic" and "gen…

…erics_upper_bound". (#1720)

These tests attempted to validate that type checkers enforced the rule that constraints and upper bounds within a TypedDict definition cannot be parameterized by other type variables (i.e. cannot be generic). The way the tests were written, they weren't actually testing this check. This allowed mypy and pyre (which both fail to catch this condition) to pass the tests. I've updated the test so it properly tests this condition.

conformance/results/mypy/generics_basic.toml

@@ -1,20 +1,21 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Does not reject the use of a constraint parameterized by another type variable.
+"""
 output = """
-generics_basic.py:36: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
-generics_basic.py:37: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
-generics_basic.py:44: error: TypeVar cannot have only a single constraint  [misc]
-generics_basic.py:48: error: Type variable "generics_basic.T" is unbound  [valid-type]
-generics_basic.py:48: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
-generics_basic.py:48: note: (Hint: Use "T" in function signature to bind "T" inside a function)
-generics_basic.py:59: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
-generics_basic.py:107: error: Duplicate type variables in Generic[...] or Protocol[...]  [misc]
-generics_basic.py:140: error: Invalid index type "int" for "MyMap1[str, int]"; expected type "str"  [index]
-generics_basic.py:141: error: Invalid index type "int" for "MyMap2[int, str]"; expected type "str"  [index]
-generics_basic.py:167: error: Dynamic metaclass not supported for "GenericMetaInstance"  [misc]
-generics_basic.py:167: error: Type variable "generics_basic.T" is unbound  [valid-type]
-generics_basic.py:167: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
-generics_basic.py:167: note: (Hint: Use "T" in function signature to bind "T" inside a function)
+generics_basic.py:40: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
+generics_basic.py:41: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
+generics_basic.py:49: error: TypeVar cannot have only a single constraint  [misc]
+generics_basic.py:69: error: Value of type variable "AnyStr" of "concat" cannot be "Sequence[object]"  [type-var]
+generics_basic.py:121: error: Duplicate type variables in Generic[...] or Protocol[...]  [misc]
+generics_basic.py:157: error: Invalid index type "int" for "MyMap1[str, int]"; expected type "str"  [index]
+generics_basic.py:158: error: Invalid index type "int" for "MyMap2[int, str]"; expected type "str"  [index]
+generics_basic.py:191: error: Dynamic metaclass not supported for "GenericMetaInstance"  [misc]
+generics_basic.py:191: error: Type variable "generics_basic.T" is unbound  [valid-type]
+generics_basic.py:191: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
+generics_basic.py:191: note: (Hint: Use "T" in function signature to bind "T" inside a function)
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 55: Expected 1 errors
 """

conformance/results/mypy/generics_upper_bound.toml

@@ -1,12 +1,13 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Does not reject use of type variable within an upper bound.
+"""
 output = """
-generics_upper_bound.py:22: error: Type variable "generics_upper_bound.T" is unbound  [valid-type]
-generics_upper_bound.py:22: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
-generics_upper_bound.py:22: note: (Hint: Use "T" in function signature to bind "T" inside a function)
-generics_upper_bound.py:41: error: Expression is of type "Collection[int]", not "list[int] | set[int]"  [assert-type]
-generics_upper_bound.py:48: error: Value of type variable "ST" of "longer" cannot be "int"  [type-var]
-generics_upper_bound.py:53: error: TypeVar cannot have both values and an upper bound  [misc]
+generics_upper_bound.py:43: error: Expression is of type "Collection[int]", not "list[int] | set[int]"  [assert-type]
+generics_upper_bound.py:51: error: Value of type variable "ST" of "longer" cannot be "int"  [type-var]
+generics_upper_bound.py:56: error: TypeVar cannot have both values and an upper bound  [misc]
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 24: Expected 1 errors
 """

conformance/results/mypy/version.toml

@@ -1,2 +1,2 @@
 version = "mypy 1.9.0"
-test_duration = 1.6
+test_duration = 1.5

conformance/results/pyre/generics_basic.toml

@@ -1,26 +1,27 @@
 conformant = "Partial"
 notes = """
 False positives in examples using constrained type variables.
+False negative for constraint parameterized by a type variable.
 False negative in custom map example.
 False positive using `iter`.
 False negative for generic metaclass.
 """
 output = """
-generics_basic.py:31:4 Incompatible return type [7]: Expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
-generics_basic.py:31:15 Incompatible parameter type [6]: In call `bytes.__add__`, for 1st positional argument, expected `Union[array[typing.Any], bytearray, bytes, _CData, memoryview, mmap, PickleBuffer]` but got `Variable[AnyStr <: [str, bytes]]`.
-generics_basic.py:36:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
-generics_basic.py:37:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `str`.
-generics_basic.py:44:0 Invalid type [31]: TypeVar can't have a single explicit constraint. Did you mean `bound=str`?
-generics_basic.py:48:0 Invalid type [31]: Expression `Variable[BadConstraint2 <: [str, Variable[generics_basic.T]]]` is not a valid type. Type variables cannot contain other type variables in their constraints.
-generics_basic.py:59:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
-generics_basic.py:107:0 Duplicate type variables [59]: Duplicate type variable `T` in Generic[...].
-generics_basic.py:161:25 Undefined attribute [16]: `typing.Iterator` has no attribute `__getitem__`.
+generics_basic.py:34:4 Incompatible return type [7]: Expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
+generics_basic.py:34:15 Incompatible parameter type [6]: In call `bytes.__add__`, for 1st positional argument, expected `Union[array[typing.Any], bytearray, bytes, _CData, memoryview, mmap, PickleBuffer]` but got `Variable[AnyStr <: [str, bytes]]`.
+generics_basic.py:40:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
+generics_basic.py:41:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `str`.
+generics_basic.py:49:0 Invalid type [31]: TypeVar can't have a single explicit constraint. Did you mean `bound=str`?
+generics_basic.py:55:57 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T]]` but got `TypeVar`.
+generics_basic.py:69:14 Incompatible parameter type [6]: In call `concat`, for 2nd positional argument, expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.
+generics_basic.py:121:0 Duplicate type variables [59]: Duplicate type variable `T` in Generic[...].
+generics_basic.py:182:25 Undefined attribute [16]: `typing.Iterator` has no attribute `__getitem__`.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 140: Expected 1 errors
-Line 141: Expected 1 errors
-Line 167: Expected 1 errors
-Line 31: Unexpected errors ['generics_basic.py:31:4 Incompatible return type [7]: Expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.', 'generics_basic.py:31:15 Incompatible parameter type [6]: In call `bytes.__add__`, for 1st positional argument, expected `Union[array[typing.Any], bytearray, bytes, _CData, memoryview, mmap, PickleBuffer]` but got `Variable[AnyStr <: [str, bytes]]`.']
-Line 161: Unexpected errors ['generics_basic.py:161:25 Undefined attribute [16]: `typing.Iterator` has no attribute `__getitem__`.']
+Line 157: Expected 1 errors
+Line 158: Expected 1 errors
+Line 191: Expected 1 errors
+Line 34: Unexpected errors ['generics_basic.py:34:4 Incompatible return type [7]: Expected `Variable[AnyStr <: [str, bytes]]` but got `bytes`.', 'generics_basic.py:34:15 Incompatible parameter type [6]: In call `bytes.__add__`, for 1st positional argument, expected `Union[array[typing.Any], bytearray, bytes, _CData, memoryview, mmap, PickleBuffer]` but got `Variable[AnyStr <: [str, bytes]]`.']
+Line 182: Unexpected errors ['generics_basic.py:182:25 Undefined attribute [16]: `typing.Iterator` has no attribute `__getitem__`.']
 """

conformance/results/pyre/generics_upper_bound.toml

@@ -3,11 +3,11 @@ notes = """
 Does not reject use of upper bound with constrained TypeVar.
 """
 output = """
-generics_upper_bound.py:22:0 Invalid type [31]: Expression `Variable[T_Bad1 (bound to typing.List[Variable[generics_upper_bound.T]])]` is not a valid type. Type variables cannot contain other type variables in their constraints.
-generics_upper_bound.py:48:7 Incompatible parameter type [6]: In call `longer`, for 1st positional argument, expected `Variable[ST (bound to Sized)]` but got `int`.
-generics_upper_bound.py:48:10 Incompatible parameter type [6]: In call `longer`, for 2nd positional argument, expected `Variable[ST (bound to Sized)]` but got `int`.
+generics_upper_bound.py:24:42 Incompatible parameter type [6]: In call `typing.GenericMeta.__getitem__`, for 1st positional argument, expected `Type[Variable[_T]]` but got `TypeVar`.
+generics_upper_bound.py:51:7 Incompatible parameter type [6]: In call `longer`, for 1st positional argument, expected `Variable[ST (bound to Sized)]` but got `int`.
+generics_upper_bound.py:51:10 Incompatible parameter type [6]: In call `longer`, for 2nd positional argument, expected `Variable[ST (bound to Sized)]` but got `int`.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 53: Expected 1 errors
+Line 56: Expected 1 errors
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.19"
-test_duration = 4.0
+test_duration = 3.2

conformance/results/pyright/generics_basic.toml

@@ -1,20 +1,19 @@
 conformant = "Pass"
 output = """
-generics_basic.py:36:15 - error: Argument of type "bytes" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
+generics_basic.py:40:15 - error: Argument of type "bytes" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
   "bytes" is incompatible with "str" (reportArgumentType)
-generics_basic.py:37:15 - error: Argument of type "str" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
+generics_basic.py:41:15 - error: Argument of type "str" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
   "str" is incompatible with "bytes" (reportArgumentType)
-generics_basic.py:44:44 - error: TypeVar must have at least two constrained types (reportGeneralTypeIssues)
-generics_basic.py:48:49 - error: Type variable "T" has no meaning in this context (reportGeneralTypeIssues)
-generics_basic.py:48:49 - error: TypeVar constraint type cannot be generic
-generics_basic.py:59:15 - error: Argument of type "bytes" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
+generics_basic.py:49:44 - error: TypeVar must have at least two constrained types (reportGeneralTypeIssues)
+generics_basic.py:55:53 - error: TypeVar constraint type cannot be generic
+generics_basic.py:69:15 - error: Argument of type "bytes" cannot be assigned to parameter "y" of type "AnyStr@concat" in function "concat"
   "bytes" is incompatible with "str" (reportArgumentType)
-generics_basic.py:107:24 - error: Type arguments for "Generic" must be unique
-generics_basic.py:140:5 - error: Argument of type "Literal[0]" cannot be assigned to parameter "key" of type "str" in function "__getitem__"
+generics_basic.py:121:24 - error: Type arguments for "Generic" must be unique
+generics_basic.py:157:5 - error: Argument of type "Literal[0]" cannot be assigned to parameter "key" of type "str" in function "__getitem__"
   "Literal[0]" is incompatible with "str" (reportArgumentType)
-generics_basic.py:141:5 - error: Argument of type "Literal[0]" cannot be assigned to parameter "key" of type "str" in function "__getitem__"
+generics_basic.py:158:5 - error: Argument of type "Literal[0]" cannot be assigned to parameter "key" of type "str" in function "__getitem__"
   "Literal[0]" is incompatible with "str" (reportArgumentType)
-generics_basic.py:167:37 - error: Metaclass cannot be generic (reportGeneralTypeIssues)
+generics_basic.py:191:37 - error: Metaclass cannot be generic (reportGeneralTypeIssues)
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/pyright/generics_upper_bound.toml

@@ -1,16 +1,15 @@
 conformant = "Pass"
 output = """
-generics_upper_bound.py:22:34 - error: TypeVar bound type cannot be generic
-generics_upper_bound.py:22:39 - error: Type variable "T" has no meaning in this context (reportGeneralTypeIssues)
-generics_upper_bound.py:48:8 - error: Argument of type "Literal[3]" cannot be assigned to parameter "x" of type "ST@longer" in function "longer"
+generics_upper_bound.py:24:38 - error: TypeVar bound type cannot be generic
+generics_upper_bound.py:51:8 - error: Argument of type "Literal[3]" cannot be assigned to parameter "x" of type "ST@longer" in function "longer"
   Type "Literal[3]" is incompatible with type "Sized"
     "Literal[3]" is incompatible with protocol "Sized"
       "__len__" is not present (reportArgumentType)
-generics_upper_bound.py:48:11 - error: Argument of type "Literal[3]" cannot be assigned to parameter "y" of type "ST@longer" in function "longer"
+generics_upper_bound.py:51:11 - error: Argument of type "Literal[3]" cannot be assigned to parameter "y" of type "ST@longer" in function "longer"
   Type "Literal[3]" is incompatible with type "Sized"
     "Literal[3]" is incompatible with protocol "Sized"
       "__len__" is not present (reportArgumentType)
-generics_upper_bound.py:53:44 - error: TypeVar cannot be both bound and constrained
+generics_upper_bound.py:56:44 - error: TypeVar cannot be both bound and constrained
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/pytype/generics_basic.toml

@@ -4,27 +4,27 @@ False positives in examples using constrained type variables.
 False negative for generic metaclass.
 """
 output = """
-File "generics_basic.py", line 31, in concat: bad return type [bad-return-type]
+File "generics_basic.py", line 34, in concat: bad return type [bad-return-type]
 Called from (traceback):
-  line 57, in test_concat_subtype
-File "generics_basic.py", line 36, in test_concat: Function concat was called with the wrong arguments [wrong-arg-types]
-File "generics_basic.py", line 37, in test_concat: Function concat was called with the wrong arguments [wrong-arg-types]
-File "generics_basic.py", line 44, in <module>: Invalid TypeVar: the number of constraints must be 0 or more than 1 [invalid-typevar]
-File "generics_basic.py", line 48, in <module>: Invalid TypeVar: constraint cannot contain TypeVars [invalid-typevar]
-File "generics_basic.py", line 57, in test_concat_subtype: MyStr [assert-type]
-File "generics_basic.py", line 58, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]
-File "generics_basic.py", line 58, in test_concat_subtype: Any [assert-type]
-File "generics_basic.py", line 59, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]
-File "generics_basic.py", line 107, in <module>: Invalid type annotation 'Generic'  [invalid-annotation]
-File "generics_basic.py", line 140, in test_my_map: unsupported operand type(s) for item retrieval: MyMap1[str, int] and int [unsupported-operands]
-File "generics_basic.py", line 141, in test_my_map: unsupported operand type(s) for item retrieval: MyMap2[int, str] and int [unsupported-operands]
-File "generics_basic.py", line 161, in test_my_iterable_any: Iterator[nothing] [assert-type]
+  line 67, in test_concat_subtype
+File "generics_basic.py", line 40, in test_concat: Function concat was called with the wrong arguments [wrong-arg-types]
+File "generics_basic.py", line 41, in test_concat: Function concat was called with the wrong arguments [wrong-arg-types]
+File "generics_basic.py", line 49, in <module>: Invalid TypeVar: the number of constraints must be 0 or more than 1 [invalid-typevar]
+File "generics_basic.py", line 55, in Test: Invalid TypeVar: constraint cannot contain TypeVars [invalid-typevar]
+File "generics_basic.py", line 67, in test_concat_subtype: MyStr [assert-type]
+File "generics_basic.py", line 68, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]
+File "generics_basic.py", line 68, in test_concat_subtype: Any [assert-type]
+File "generics_basic.py", line 69, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]
+File "generics_basic.py", line 121, in <module>: Invalid type annotation 'Generic'  [invalid-annotation]
+File "generics_basic.py", line 157, in test_my_map: unsupported operand type(s) for item retrieval: MyMap1[str, int] and int [unsupported-operands]
+File "generics_basic.py", line 158, in test_my_map: unsupported operand type(s) for item retrieval: MyMap2[int, str] and int [unsupported-operands]
+File "generics_basic.py", line 182, in test_my_iterable_any: Iterator[nothing] [assert-type]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 167: Expected 1 errors
-Line 31: Unexpected errors ['File "generics_basic.py", line 31, in concat: bad return type [bad-return-type]']
-Line 57: Unexpected errors ['File "generics_basic.py", line 57, in test_concat_subtype: MyStr [assert-type]']
-Line 58: Unexpected errors ['File "generics_basic.py", line 58, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]', 'File "generics_basic.py", line 58, in test_concat_subtype: Any [assert-type]']
-Line 161: Unexpected errors ['File "generics_basic.py", line 161, in test_my_iterable_any: Iterator[nothing] [assert-type]']
+Line 191: Expected 1 errors
+Line 34: Unexpected errors ['File "generics_basic.py", line 34, in concat: bad return type [bad-return-type]']
+Line 67: Unexpected errors ['File "generics_basic.py", line 67, in test_concat_subtype: MyStr [assert-type]']
+Line 68: Unexpected errors ['File "generics_basic.py", line 68, in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]', 'File "generics_basic.py", line 68, in test_concat_subtype: Any [assert-type]']
+Line 182: Unexpected errors ['File "generics_basic.py", line 182, in test_my_iterable_any: Iterator[nothing] [assert-type]']
 """

conformance/results/pytype/generics_upper_bound.toml

@@ -3,15 +3,15 @@ notes = """
 Does not properly support assert_type.
 """
 output = """
-File "generics_upper_bound.py", line 22, in <module>: Invalid TypeVar: bound cannot contain TypeVars [invalid-typevar]
-File "generics_upper_bound.py", line 35, in <module>: list [assert-type]
-File "generics_upper_bound.py", line 36, in <module>: set [assert-type]
-File "generics_upper_bound.py", line 41, in <module>: Union[list, set] [assert-type]
-File "generics_upper_bound.py", line 48, in <module>: Function longer was called with the wrong arguments [wrong-arg-types]
-File "generics_upper_bound.py", line 53, in <module>: Invalid TypeVar: constraints and a bound are mutually exclusive [invalid-typevar]
+File "generics_upper_bound.py", line 24, in Test: Invalid TypeVar: bound cannot contain TypeVars [invalid-typevar]
+File "generics_upper_bound.py", line 37, in <module>: list [assert-type]
+File "generics_upper_bound.py", line 38, in <module>: set [assert-type]
+File "generics_upper_bound.py", line 43, in <module>: Union[list, set] [assert-type]
+File "generics_upper_bound.py", line 51, in <module>: Function longer was called with the wrong arguments [wrong-arg-types]
+File "generics_upper_bound.py", line 56, in <module>: Invalid TypeVar: constraints and a bound are mutually exclusive [invalid-typevar]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 35: Unexpected errors ['File "generics_upper_bound.py", line 35, in <module>: list [assert-type]']
-Line 36: Unexpected errors ['File "generics_upper_bound.py", line 36, in <module>: set [assert-type]']
+Line 37: Unexpected errors ['File "generics_upper_bound.py", line 37, in <module>: list [assert-type]']
+Line 38: Unexpected errors ['File "generics_upper_bound.py", line 38, in <module>: set [assert-type]']
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.04.11"
-test_duration = 37.5
+test_duration = 35.1