Merge pull request #1036 from google/google_sync

Google sync

CHANGELOG

@@ -1,3 +1,21 @@
+Version 2021.10.25:
+
+New features and updates:
+* Completely remove the --preserve-union-macros flag. This was a no-op as of
+  version 2021.10.17.
+* Add a new flag, --build-dict-literals-from-kwargs, to construct dict literals
+  for `dict(k=v, ...)`. This behavior will ultimately by enabled by default and
+  the flag removed.
+* Add a new flag, --strict_namedtuple_checks, for typing.NamedTuple and
+  collections.namedtuple to inherit from fixed-length tuples. This behavior will
+  ultimately be enabled by default and the flag removed.
+
+Bug fixes:
+* Fix how the enum overlay calls `base_type.__new__`.
+* Improve how the enum overlay chooses base types.
+* When an `attr.ib()` call has type errors, construct the attrib anyway.
+* Support builtin str removeprefix/removesuffix.
+
 Version 2021.10.18
 
 New features and updates:
@@ -11,6 +29,8 @@ Bug fixes:
 * Enum overlay: use ClassVar to differentiate enum class attributes.
 * Fix a parser bug involving nested class name conflicts.
 * Fix a crash when lazily loading a missing submodule.
+* Change PrintVisitor to group explicit imports with the ones collected from
+  type annotations.
 
 Version 2021.10.11
 

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2021.10.18'
+__version__ = '2021.10.25'

pytype/config.py

@@ -167,6 +167,16 @@ def add_basic_options(o):
       "--allow-recursive-types", action="store_true",
       dest="allow_recursive_types", default=False,
       help="Allow recursive type definitions. " + temporary)
+  o.add_argument(
+      "--build-dict-literals-from-kwargs", action="store_true",
+      dest="build_dict_literals_from_kwargs", default=False,
+      help="Build dict literals from dict(k=v, ...) calls. " + temporary)
+  o.add_argument(
+      "--strict_namedtuple_checks", action="store_true",
+      dest="strict_namedtuple_checks", default=False,
+      help=(
+          "Enable stricter namedtuple checks, such as unpacking and "
+          "'typing.Tuple' compatibility. ") + temporary)
 
 
 def add_subtools(o):

pytype/matcher.py

@@ -806,19 +806,21 @@ def _match_instance(self, left, instance, other_type, subst, view):
   def _match_maybe_parameterized_instance(self, left, instance, other_type,
                                           subst, view):
     """Used by _match_instance."""
+    def assert_classes_match(cls1, cls2):
+      # We need the somewhat complex assertion below to allow internal
+      # subclasses of abstract classes to act as their parent classes.
+      assert isinstance(cls1, type(cls2)) or isinstance(cls2, type(cls1))
+
     if isinstance(other_type, abstract.ParameterizedClass):
       if isinstance(left, abstract.ParameterizedClass):
-        assert left.base_cls is other_type.base_cls
+        assert_classes_match(left.base_cls, other_type.base_cls)
       elif isinstance(left, abstract.AMBIGUOUS_OR_EMPTY):
         return self._subst_with_type_parameters_from(subst, other_type)
       else:
         # Parameterized classes can rename type parameters, which is why we need
         # the instance type for lookup. But if the instance type is not
         # parameterized, then it is safe to use the param names in other_type.
-        # We need the somewhat complex assertion below to allow internal
-        # subclasses of abstract classes to act as their parent classes.
-        assert (isinstance(left, type(other_type.base_cls)) or
-                isinstance(other_type.base_cls, type(left)))
+        assert_classes_match(left, other_type.base_cls)
         left = other_type
       for type_param in left.template:
         class_param = other_type.get_formal_type_parameter(type_param.name)

pytype/overlays/collections_overlay.py

@@ -18,18 +18,27 @@ def _repeat_type(type_str, n):
   return ", ".join((type_str,) * n) if n else "()"
 
 
-def namedtuple_ast(name, fields, defaults, python_version=None):
+def namedtuple_ast(name,
+                   fields,
+                   defaults,
+                   python_version=None,
+                   strict_namedtuple_checks=True):
   """Make an AST with a namedtuple definition for the given name and fields.
 
   Args:
     name: The namedtuple name.
     fields: The namedtuple fields.
     defaults: Sequence of booleans, whether each field has a default.
     python_version: Optionally, the python version of the code under analysis.
+    strict_namedtuple_checks: Whether to enable a stricter type annotation
+      hierarchy for generated NamedType. e.g. Tuple[n*[Any]] instead of tuple.
+      This should usually be set to the value of
+        ctx.options.strict_namedtuple_checks
 
   Returns:
     A pytd.TypeDeclUnit with the namedtuple definition in its classes.
   """
+
   typevar = visitors.CreateTypeParametersForSignatures.PREFIX + name
   num_fields = len(fields)
   field_defs = "\n  ".join(
@@ -41,7 +50,7 @@ def namedtuple_ast(name, fields, defaults, python_version=None):
 
   nt = textwrap.dedent("""
     {typevar} = TypeVar("{typevar}", bound={name})
-    class {name}(tuple):
+    class {name}({tuple_superclass_type}):
       __dict__ = ...  # type: collections.OrderedDict[str, typing.Any]
       __slots__ = [{field_names_as_strings}]
       _fields = ...  # type: typing.Tuple[{repeat_str}]
@@ -59,13 +68,17 @@ def _make(cls: typing.Type[{typevar}],
                 len: typing.Callable[[typing.Sized], int] = ...
       ) -> {typevar}: ...
       def _replace(self: {typevar}, **kwds) -> {typevar}: ...
-  """).format(typevar=typevar,
-              name=name,
-              repeat_str=_repeat_type("str", num_fields),
-              field_defs=field_defs,
-              repeat_any=_repeat_type("typing.Any", num_fields),
-              fields_as_parameters=fields_as_parameters,
-              field_names_as_strings=field_names_as_strings)
+  """).format(
+      typevar=typevar,
+      name=name,
+      repeat_str=_repeat_type("str", num_fields),
+      tuple_superclass_type=("typing.Tuple[{}]".format(
+          _repeat_type("typing.Any", num_fields))
+                             if strict_namedtuple_checks else "tuple"),
+      field_defs=field_defs,
+      repeat_any=_repeat_type("typing.Any", num_fields),
+      fields_as_parameters=fields_as_parameters,
+      field_names_as_strings=field_names_as_strings)
   return parser.parse_string(nt, python_version=python_version)
 
 
@@ -286,7 +299,11 @@ class have to be changed to match the number and names of the fields, we
 
     name = escape.pack_namedtuple(name, field_names)
     ast = namedtuple_ast(
-        name, field_names, defaults, python_version=self.ctx.python_version)
+        name,
+        field_names,
+        defaults,
+        python_version=self.ctx.python_version,
+        strict_namedtuple_checks=self.ctx.options.strict_namedtuple_checks)
     mapping = self._get_known_types_mapping()
 
     # A truly well-formed pyi for the namedtuple will have references to the new

pytype/overlays/collections_overlay_test.py

@@ -14,7 +14,10 @@ class NamedTupleAstTest(test_base.UnitTest):
 
   def _namedtuple_ast(self, name, fields):
     return collections_overlay.namedtuple_ast(
-        name, fields, [False] * len(fields), self.python_version)
+        name,
+        fields, [False] * len(fields),
+        self.python_version,
+        strict_namedtuple_checks=True)
 
   def test_basic(self):
     ast = self._namedtuple_ast("X", [])

pytype/overlays/typing_overlay.py

@@ -377,6 +377,12 @@ def _build_namedtuple(self, name, field_names, field_types, node, bases):
         varargs=Param("args"),
         kwargs=Param("kwargs"))
 
+    heterogeneous_tuple_type_params = dict(enumerate(field_types))
+    heterogeneous_tuple_type_params[abstract_utils.T] = field_types_union
+    # Representation of the to-be-created NamedTuple as a typing.Tuple.
+    heterogeneous_tuple_type = abstract.TupleClass(
+        self.ctx.convert.tuple_type, heterogeneous_tuple_type_params, self.ctx)
+
     # _make
     # _make is a classmethod, so it needs to be wrapped by
     # specialibuiltins.ClassMethodInstance.
@@ -422,13 +428,11 @@ def _build_namedtuple(self, name, field_names, field_types, node, bases):
         kwargs=Param("kwds", field_types_union))
 
     # __getnewargs__
-    getnewargs_tuple_params = dict(
-        tuple(enumerate(field_types)) +
-        ((abstract_utils.T, field_types_union),))
-    getnewargs_tuple = abstract.TupleClass(self.ctx.convert.tuple_type,
-                                           getnewargs_tuple_params, self.ctx)
     members["__getnewargs__"] = overlay_utils.make_method(
-        self.ctx, node, name="__getnewargs__", return_type=getnewargs_tuple)
+        self.ctx,
+        node,
+        name="__getnewargs__",
+        return_type=heterogeneous_tuple_type)
 
     # __getstate__
     members["__getstate__"] = overlay_utils.make_method(
@@ -442,15 +446,20 @@ def _build_namedtuple(self, name, field_names, field_types, node, bases):
     cls_dict = abstract.Dict(self.ctx)
     cls_dict.update(node, members)
 
+    if self.ctx.options.strict_namedtuple_checks:
+      # Enforces type checking like Tuple[...]
+      superclass_of_new_type = heterogeneous_tuple_type.to_variable(node)
+    else:
+      superclass_of_new_type = self.ctx.convert.tuple_type.to_variable(node)
     if bases:
       final_bases = []
       for base in bases:
         if any(b.full_name == "typing.NamedTuple" for b in base.data):
-          final_bases.append(self.ctx.convert.tuple_type.to_variable(node))
+          final_bases.append(superclass_of_new_type)
         else:
           final_bases.append(base)
     else:
-      final_bases = [self.ctx.convert.tuple_type.to_variable(node)]
+      final_bases = [superclass_of_new_type]
       # This NamedTuple is being created via a function call. We manually
       # construct an annotated_locals entry for it so that __annotations__ is
       # initialized properly for the generated class.

pytype/special_builtins.py

@@ -704,7 +704,11 @@ def __init__(self, ctx):
     super().__init__(ctx, "dict")
 
   def call(self, node, funcb, args):
-    if not args.has_non_namedargs():
+    if self.ctx.options.build_dict_literals_from_kwargs:
+      build_literal = not args.has_non_namedargs()
+    else:
+      build_literal = args.is_empty()
+    if build_literal:
       # special-case a dict constructor with explicit k=v args
       d = abstract.Dict(self.ctx)
       for (k, v) in args.namedargs.items():

pytype/tests/test_base.py

@@ -132,6 +132,8 @@ def setUp(self):
     self.options = config.Options.create(python_version=self.python_version,
                                          allow_recursive_types=True,
                                          bind_properties=True,
+                                         build_dict_literals_from_kwargs=True,
+                                         strict_namedtuple_checks=True,
                                          use_enum_overlay=True)
 
   @property

pytype/tests/test_builtins1.py

@@ -483,7 +483,9 @@ class Foo(
     """)
     name = escape.pack_namedtuple("_Foo", ["x", "y", "z"])
     ast = collections_overlay.namedtuple_ast(
-        name, ["x", "y", "z"], [False] * 3, self.python_version)
+        name, ["x", "y", "z"], [False] * 3,
+        self.python_version,
+        strict_namedtuple_checks=self.options.strict_namedtuple_checks)
     expected = pytd_utils.Print(ast) + textwrap.dedent("""
       collections = ...  # type: module
       class Foo({name}): ...""").format(name=name)
@@ -502,7 +504,9 @@ def test_store_and_load_from_namedtuple(self):
     """)
     name = escape.pack_namedtuple("t", ["x", "y", "z"])
     ast = collections_overlay.namedtuple_ast(
-        name, ["x", "y", "z"], [False] * 3, self.python_version)
+        name, ["x", "y", "z"], [False] * 3,
+        self.python_version,
+        strict_namedtuple_checks=self.options.strict_namedtuple_checks)
     expected = pytd_utils.Print(ast) + textwrap.dedent("""
       collections = ...  # type: module
       t = {name}

pytype/tests/test_functions1.py

@@ -35,7 +35,7 @@ def test_empty_vs_deleted(self):
       import collections
       Foo = collections.namedtuple('Foo', 'x')
       def f():
-        x, _ = Foo(10)  # x gets set to abstract.Empty here.
+        (x,) = Foo(10)  # x gets set to abstract.Empty here.
         def g():
           return x  # Should not raise a name-error
     """)

pytype/tests/test_namedtuple1.py

@@ -14,7 +14,10 @@ class NamedtupleTests(test_base.BaseTest):
 
   def _namedtuple_ast(self, name, fields):
     return collections_overlay.namedtuple_ast(
-        name, fields, [False] * len(fields), self.python_version)
+        name,
+        fields, [False] * len(fields),
+        python_version=self.python_version,
+        strict_namedtuple_checks=self.options.strict_namedtuple_checks)
 
   def _namedtuple_def(self, suffix="", **kws):
     """Generate the expected pyi for a simple namedtuple definition.
@@ -163,6 +166,18 @@ def test_unpacking(self):
         a_f, b_f, c_f = a
         """)
 
+  def test_bad_unpacking(self):
+    self.CheckWithErrors(
+        """
+        import collections
+        X = collections.namedtuple("X", "a b c")
+
+        a = X(1, "2", 42.0)
+
+        _, _, _, too_many = a  # bad-unpacking
+        _, too_few = a  # bad-unpacking
+        """)
+
   def test_is_tuple_and_superclasses(self):
     """Test that a collections.namedtuple behaves like a tuple typewise."""
     self.Check(

pytype/tests/test_typing_namedtuple1.py

@@ -230,8 +230,10 @@ def test_is_not_incorrect_types(self):
 
       x = X(1, "2")
 
+      x_wrong_tuple_types = x  # type: Tuple[str, str]  # annotation-type-mismatch
       x_not_a_list = x  # type: list  # annotation-type-mismatch
       x_not_a_mutable_seq = x  # type: MutableSequence[Union[int, str]]  # annotation-type-mismatch
+      x_first_wrong_element_type = x[0]  # type: str  # annotation-type-mismatch
     """)
 
   def test_meets_protocol(self):
@@ -292,7 +294,7 @@ def test_generated_members(self):
 
         _TX = TypeVar('_TX', bound=X)
 
-        class X(tuple):
+        class X(Tuple[int, str]):
             __slots__ = ["a", "b"]
             __dict__: collections.OrderedDict[str, Union[int, str]]
             _field_defaults: collections.OrderedDict[str, Union[int, str]]

pytype/tests/test_typing_namedtuple2.py

@@ -85,7 +85,7 @@ def test_basic_namedtuple(self):
         a = ...  # type: int
         b = ...  # type: str
         _TX = TypeVar('_TX', bound=X)
-        class X(tuple):
+        class X(Tuple[int, str]):
           __slots__ = ["a", "b"]
           __dict__: collections.OrderedDict[str, Union[int, str]]
           _field_defaults: collections.OrderedDict[str, Union[int, str]]
@@ -212,7 +212,7 @@ class SubNamedTuple(baseClass, NamedTuple):
 
         _TSubNamedTuple = TypeVar('_TSubNamedTuple', bound=SubNamedTuple)
 
-        class SubNamedTuple(baseClass, tuple):
+        class SubNamedTuple(baseClass, Tuple[int]):
             __slots__ = ["a"]
             __dict__ = ...  # type: collections.OrderedDict[str, int]
             _field_defaults = ...  # type: collections.OrderedDict[str, int]
@@ -329,8 +329,10 @@ class X(NamedTuple):
 
       x = X(1, "2")
 
+      x_wrong_tuple_types = x  # type: Tuple[str, str]  # annotation-type-mismatch
       x_not_a_list = x  # type: list  # annotation-type-mismatch
       x_not_a_mutable_seq = x  # type: MutableSequence[Union[int, str]]  # annotation-type-mismatch
+      x_first_wrong_element_type = x[0]  # type: str  # annotation-type-mismatch
     """)
 
   def test_meets_protocol(self):
@@ -398,7 +400,7 @@ class X(NamedTuple):
 
         _TX = TypeVar('_TX', bound=X)
 
-        class X(tuple):
+        class X(Tuple[int, str]):
             __slots__ = ["a", "b"]
             __dict__: collections.OrderedDict[str, Union[int, str]]
             _field_defaults: collections.OrderedDict[str, Union[int, str]]
@@ -463,7 +465,7 @@ def func():
 
         _TSubNamedTuple = TypeVar('_TSubNamedTuple', bound=SubNamedTuple)
 
-        class SubNamedTuple(tuple):
+        class SubNamedTuple(Tuple[int, str, int]):
             __slots__ = ["a", "b", "c"]
             __dict__: collections.OrderedDict[str, Union[int, str]]
             _field_defaults: collections.OrderedDict[str, Union[int, str]]
@@ -548,7 +550,7 @@ def _int_identity(x: int) -> int: ...
 
         _TFoo = TypeVar('_TFoo', bound=Foo)
 
-        class Foo(tuple, Generic[T]):
+        class Foo(Tuple[T, Callable[[T], T]], Generic[T]):
           __slots__ = ["x", "y"]
           # TODO(csyoung): Figure out why these two fields' value types are
           # being collapsed to Any.

pytype/tools/analyze_project/config.py

@@ -67,6 +67,9 @@
         None, 'False', ArgInfo('--use-enum-overlay', None), None),
     'allow_recursive_types': Item(
         None, 'False', ArgInfo('--use-recursive-types', None), None),
+    'build_dict_literals_from_kwargs': Item(
+        None, 'False', ArgInfo('--build-dict-literals-from-kwargs', None),
+        None),
     'disable': Item(
         None, 'pyi-error', ArgInfo('--disable', ','.join),
         'Comma or space separated list of error names to ignore.'),
@@ -77,6 +80,8 @@
     'protocols': Item(None, 'False', ArgInfo('--protocols', None), None),
     'strict_import': Item(
         None, 'False', ArgInfo('--strict-import', None), None),
+    'strict_namedtuple_checks': Item(
+        None, 'False', ArgInfo('--strict_namedtuple_checks', None), None),
 }