Merge pull request #1017 from google/google_sync

Google sync

pytype/overlays/collections_overlay.py

@@ -18,12 +18,13 @@ def _repeat_type(type_str, n):
   return ", ".join((type_str,) * n) if n else "()"
 
 
-def namedtuple_ast(name, fields, python_version=None):
+def namedtuple_ast(name, fields, defaults, python_version=None):
   """Make an AST with a namedtuple definition for the given name and fields.
 
   Args:
     name: The namedtuple name.
     fields: The namedtuple fields.
+    defaults: Sequence of booleans, whether each field has a default.
     python_version: Optionally, the python version of the code under analysis.
 
   Returns:
@@ -33,7 +34,9 @@ def namedtuple_ast(name, fields, python_version=None):
   num_fields = len(fields)
   field_defs = "\n  ".join(
       "%s = ...  # type: typing.Any" % field for field in fields)
-  field_names = "".join(", " + field for field in fields)
+  fields_as_parameters = "".join(
+      ", " + field + (" = ..." if default else "")
+      for field, default in zip(fields, defaults))
   field_names_as_strings = ", ".join(repr(field) for field in fields)
 
   nt = textwrap.dedent("""
@@ -46,7 +49,8 @@ class {name}(tuple):
       def __getnewargs__(self) -> typing.Tuple[{repeat_any}]: ...
       def __getstate__(self) -> None: ...
       def __init__(self, *args, **kwargs) -> None: ...
-      def __new__(cls: typing.Type[{typevar}]{field_names}) -> {typevar}: ...
+      def __new__(
+          cls: typing.Type[{typevar}]{fields_as_parameters}) -> {typevar}: ...
       def _asdict(self) -> collections.OrderedDict[str, typing.Any]: ...
       @classmethod
       def _make(cls: typing.Type[{typevar}],
@@ -60,7 +64,7 @@ def _replace(self: {typevar}, **kwds) -> {typevar}: ...
               repeat_str=_repeat_type("str", num_fields),
               field_defs=field_defs,
               repeat_any=_repeat_type("typing.Any", num_fields),
-              field_names=field_names,
+              fields_as_parameters=fields_as_parameters,
               field_names_as_strings=field_names_as_strings)
   return parser.parse_string(nt, python_version=python_version)
 
@@ -131,8 +135,13 @@ def _getargs(self, node, args):
       args: A function.Args object
 
     Returns:
-      A tuple containing the typename, field_names and rename arguments passed
-      to this call to collections.namedtuple.
+      A tuple containing the typename, field_names, defaults, and rename
+      arguments passed to this call to collections.namedtuple. defaults is
+      postprocessed from a sequence of defaults to a sequence of bools
+      describing whether each field has a default (e.g., for
+        collections.namedtuple('X', field_names=['a', 'b'], defaults=[0])
+      this method will return [False, True] for defaults to indicate that 'a'
+      does not have a default while 'b' does).
 
     Raises:
       function.FailedFunctionCall: The arguments do not match those needed by
@@ -168,16 +177,24 @@ def _getargs(self, node, args):
                      for f in fields]
       field_names = [utils.native_str(f) for f in field_names]
 
+    if "defaults" in callargs:
+      default_vars = abstract_utils.get_atomic_python_constant(
+          callargs["defaults"])
+      num_defaults = len(default_vars)
+      defaults = [False] * (len(fields) - num_defaults) + [True] * num_defaults
+    else:
+      defaults = [False] * len(fields)
+
     # namedtuple also takes a "verbose" argument, but we don't care about that.
 
     # rename will take any problematic field names and give them a new name.
     # Like the other args, it's stored as a Variable, but we want just a bool.
-    if callargs.get("rename", None):
+    if "rename" in callargs:
       rename = abstract_utils.get_atomic_python_constant(callargs["rename"])
     else:
       rename = False
 
-    return name_var, field_names, rename
+    return name_var, field_names, defaults, rename
 
   def _validate_and_rename_args(self, typename, field_names, rename):
     # namedtuple field names have some requirements:
@@ -247,7 +264,7 @@ class have to be changed to match the number and names of the fields, we
     """
     # If we can't extract the arguments, we take the easy way out and return Any
     try:
-      name_var, field_names, rename = self._getargs(node, args)
+      name_var, field_names, defaults, rename = self._getargs(node, args)
     except abstract_utils.ConversionError:
       return node, self.vm.new_unsolvable(node)
 
@@ -267,7 +284,7 @@ class have to be changed to match the number and names of the fields, we
       return node, self.vm.new_unsolvable(node)
 
     name = escape.pack_namedtuple(name, field_names)
-    ast = namedtuple_ast(name, field_names,
+    ast = namedtuple_ast(name, field_names, defaults,
                          python_version=self.vm.python_version)
     mapping = self._get_known_types_mapping()
 

pytype/overlays/collections_overlay_test.py

@@ -13,7 +13,8 @@ class NamedTupleAstTest(test_base.UnitTest):
   """Test collection_overlay's namedtuple AST generation."""
 
   def _namedtuple_ast(self, name, fields):
-    return collections_overlay.namedtuple_ast(name, fields, self.python_version)
+    return collections_overlay.namedtuple_ast(
+        name, fields, [False] * len(fields), self.python_version)
 
   def test_basic(self):
     ast = self._namedtuple_ast("X", [])

pytype/overlays/enum_overlay.py

@@ -489,34 +489,19 @@ def _setup_pytdclass(self, node, cls):
     # Only constants need to be transformed. We assume that enums in type
     # stubs are fully realized, i.e. there are no auto() calls and the members
     # already have values of the base type.
-    # Instance attributes are stored as properties, which pytype marks using
-    # typing.Annotated(<base_type>, 'property').
     # TODO(tsudol): Ensure only valid enum members are transformed.
-    instance_attrs = {}
-    possible_members = {}
+    member_types = []
     for pytd_val in cls.pytd_cls.constants:
       if pytd_val.name in abstract_utils.DYNAMIC_ATTRIBUTE_MARKERS:
         continue
-      assert isinstance(pytd_val, pytd.Constant)
-      if isinstance(pytd_val.type, pytd.Annotated):
-        if "'property'" in pytd_val.type.annotations:
-          instance_attrs[pytd_val.name] = pytd_val.Replace(
-              type=pytd_val.type.base_type)
-          # Properties must be deleted from the class's member map, otherwise
-          # pytype will not complain when you try to access them on the class.
-          del cls._member_map[pytd_val.name]  # pylint: disable=protected-access
-          # Note that we can't remove the entry from cls.members, because
-          # datatypes.MonitorDict does not support __delitem__. This shouldn't
-          # be an issue, because cls shouldn't have had any reason to load
-          # members by this point.
-        else:
-          possible_members[pytd_val.name] = pytd_val.Replace(
-              type=pytd_val.type.base_type)
-      else:
-        possible_members[pytd_val.name] = pytd_val
-
-    member_types = []
-    for name, pytd_val in possible_members.items():
+      # @property values are instance attributes and must not be converted into
+      # enum members. Ideally, these would only be present on the enum members,
+      # but pytype doesn't differentiate between class and instance attributes
+      # for PyTDClass and there's no mechanism to ensure canonical instances
+      # have these attributes.
+      if (isinstance(pytd_val.type, pytd.Annotated) and
+          "'property'" in pytd_val.type.annotations):
+        continue
       # Build instances directly, because you can't call instantiate() when
       # creating the class -- pytype complains about recursive types.
       member = abstract.Instance(cls, self.vm)
@@ -533,11 +518,8 @@ def _setup_pytdclass(self, node, cls):
           pyval=pytd.Constant(name="value", type=value_type),
           node=node)
       member.members["_value_"] = member.members["value"]
-      for attr_name, attr_val in instance_attrs.items():
-        member.members[attr_name] = self.vm.convert.constant_to_var(
-            pyval=attr_val, node=node)
-      cls._member_map[name] = member  # pylint: disable=protected-access
-      cls.members[name] = member.to_variable(node)
+      cls._member_map[pytd_val.name] = member  # pylint: disable=protected-access
+      cls.members[pytd_val.name] = member.to_variable(node)
       member_types.append(value_type)
     # Because we overwrite __new__, we need to mark dynamic enums here.
     # Of course, this can be moved later once custom __init__ is supported.

pytype/pyi/parser.py

@@ -311,13 +311,21 @@ def visit_Expr(self, node):
   def visit_arg(self, node):
     self.convert_node_annotations(node)
 
+  def _get_name(self, node):
+    if isinstance(node, ast3.Name):
+      return node.id
+    elif isinstance(node, ast3.Attribute):
+      return f"{node.value.id}.{node.attr}"
+    else:
+      raise ParseError(f"Unexpected node type in get_name: {node}")
+
   def _preprocess_decorator_list(self, node):
     decorators = []
     for d in node.decorator_list:
-      if isinstance(d, ast3.Name):
-        decorators.append(d.id)
-      elif isinstance(d, ast3.Attribute):
-        decorators.append(f"{d.value.id}.{d.attr}")
+      if isinstance(d, (ast3.Name, ast3.Attribute)):
+        decorators.append(self._get_name(d))
+      elif isinstance(d, ast3.Call):
+        decorators.append(self._get_name(d.func))
       else:
         raise ParseError(f"Unexpected decorator: {d}")
     node.decorator_list = decorators

pytype/pyi/parser_test.py

@@ -2083,18 +2083,6 @@ class A:
           def name(self): ...
       """, 1, "@name.setter needs 2 param(s), got 1")
 
-    self.check_error("""
-      class A:
-          @name.foo
-          def name(self): ...
-      """, 1, "Unhandled decorator: name.foo")
-
-    self.check_error("""
-      class A:
-          @notname.deleter
-          def name(self): ...
-      """, 1, "Unhandled decorator: notname.deleter")
-
     self.check_error("""
       class A:
           @property

pytype/pytd/codegen/function.py

@@ -242,7 +242,7 @@ def add_overload(self, fn: NameAndSig):
 
 def merge_method_signatures(
     name_and_sigs: List[NameAndSig],
-    check_unhandled_decorator: bool = True
+    check_unhandled_decorator: bool = False
 ) -> List[pytd.Function]:
   """Group the signatures by name, turning each group into a function."""
   functions = collections.OrderedDict()

pytype/tests/test_builtins1.py

@@ -482,8 +482,8 @@ class Foo(
         pass
     """)
     name = escape.pack_namedtuple("_Foo", ["x", "y", "z"])
-    ast = collections_overlay.namedtuple_ast(name, ["x", "y", "z"],
-                                             self.python_version)
+    ast = collections_overlay.namedtuple_ast(
+        name, ["x", "y", "z"], [False] * 3, self.python_version)
     expected = pytd_utils.Print(ast) + textwrap.dedent("""
       collections = ...  # type: module
       class Foo({name}): ...""").format(name=name)
@@ -501,8 +501,8 @@ def test_store_and_load_from_namedtuple(self):
       z = t.z
     """)
     name = escape.pack_namedtuple("t", ["x", "y", "z"])
-    ast = collections_overlay.namedtuple_ast(name, ["x", "y", "z"],
-                                             self.python_version)
+    ast = collections_overlay.namedtuple_ast(
+        name, ["x", "y", "z"], [False] * 3, self.python_version)
     expected = pytd_utils.Print(ast) + textwrap.dedent("""
       collections = ...  # type: module
       t = {name}

pytype/tests/test_enums.py

@@ -1053,8 +1053,34 @@ class NoFn(enum.Enum):
         assert_type(foo.Fn.A.x, str)
         assert_type(foo.NoFn.A.value, int)
         assert_type(foo.NoFn.A.x, str)
-        foo.Fn.x  # attribute-error
-        foo.NoFn.x  # attribute-error
+        # These should be attribute errors but pytype does not differentiate
+        # between class and instance attributes for PyTDClass.
+        foo.Fn.x
+        foo.NoFn.x
+      """, pythonpath=[d.path])
+
+  def test_instance_attrs_canonical(self):
+    # Test that canonical instances have instance attributes.
+    with file_utils.Tempdir() as d:
+      d.create_file("foo.pyi", """
+        import enum
+        from typing import Annotated
+        class F(enum.Enum):
+          A: str
+          x = Annotated[int, 'property']
+      """)
+      self.Check("""
+        import enum
+        import foo
+        class M(enum.Enum):
+          A = 'a'
+          @property
+          def x(self) -> int:
+            return 1
+        def take_f(f: foo.F):
+          return f.x
+        def take_m(m: M):
+          return m.x
       """, pythonpath=[d.path])
 
   def test_enum_bases(self):

pytype/tests/test_namedtuple1.py

@@ -13,7 +13,8 @@ class NamedtupleTests(test_base.BaseTest):
   """Tests for collections.namedtuple."""
 
   def _namedtuple_ast(self, name, fields):
-    return collections_overlay.namedtuple_ast(name, fields, self.python_version)
+    return collections_overlay.namedtuple_ast(
+        name, fields, [False] * len(fields), self.python_version)
 
   def _namedtuple_def(self, suffix="", **kws):
     """Generate the expected pyi for a simple namedtuple definition.

pytype/tests/test_namedtuple2.py

@@ -1,6 +1,7 @@
 """Tests for the namedtuple implementation in collections_overlay.py."""
 
 from pytype.tests import test_base
+from pytype.tests import test_utils
 
 
 class NamedtupleTests(test_base.BaseTest):
@@ -41,6 +42,15 @@ class Foo(NamedTuple):
         foo = Foo(x=0)
     """)
 
+  @test_utils.skipBeforePy((3, 7), "'defaults' is new in 3.7")
+  def test_namedtuple_defaults(self):
+    self.Check("""
+      import collections
+      X = collections.namedtuple('X', ['a', 'b'], defaults=[0])
+      X('a')
+      X('a', 'b')
+    """)
+
 
 if __name__ == "__main__":
   test_base.main()

pytype/tests/test_test_code.py

@@ -66,5 +66,19 @@ def test_foo(self):
           assert_type(x, int)
     """)
 
+  def test_instance_attribute(self):
+    self.Check("""
+      import unittest
+      class Foo:
+        def __init__(self, x):
+          self.x = x
+      class FooTest(unittest.TestCase):
+        def test_foo(self):
+          foo = __any_object__
+          self.assertIsInstance(foo, Foo)
+          print(foo.x)
+    """)
+
+
 if __name__ == "__main__":
   test_base.main()

pytype/vm.py

@@ -3826,7 +3826,8 @@ def _set_type_from_assert_isinstance(self, state, var, class_spec):
     # we check that at least one binding of var is compatible with typ?
     classes = []
     abstract_utils.flatten(class_spec, classes)
-    new_type = self.merge_values(classes).instantiate(state.node)
+    node, new_type = self.init_class(state.node, self.merge_values(classes))
+    state = state.change_cfg_node(node)
     return self._store_new_var_in_local(state, var, new_type)
 
   def _check_test_assert(self, state, func, args):