Merge pull request #607 from google/google_sync

Google sync

pytype/CMakeLists.txt

@@ -371,7 +371,7 @@ py_library(
     pytd/type_match.py
     pytd/visitors.py
   DEPS
-    .builtins_pytd  # TODO(sivachandra): Move this into a data dependency
+    .builtins_pytd  # TODO(b/159043305): Move this into a data dependency
     .debug
     .metrics
     .node

pytype/abstract.py

@@ -1,4 +1,3 @@
-# Lint as: python3
 """The abstract values used by vm.py.
 
 This file contains AtomicAbstractValue and its subclasses. Mixins such as Class
@@ -1108,7 +1107,6 @@ def update_slot(self, node, *args, **kwargs):
   def update(self, node, other_dict, omit=()):
     if isinstance(other_dict, (Dict, dict)):
       for key, value in other_dict.items():
-        # TODO(kramm): sources
         if key not in omit:
           self.set_str_item(node, key, value)
       if isinstance(other_dict, Dict):
@@ -1358,8 +1356,6 @@ def _build_value(self, node, raw_inner, ellipses):
         # It's a common mistake to index tuple, not tuple().
         # We only check the "int" case, since string literals are allowed for
         # late annotations.
-        # TODO(kramm): Instead of blacklisting only int, this should use
-        # annotations_util.py to look up legal types.
         if isinstance(val, Instance) and val.cls == self.vm.convert.int_type:
           # Don't report this error again.
           inner = (self.vm.convert.unsolvable,)
@@ -1758,8 +1754,7 @@ def _log_args(self, arg_values_list, level=0, logged=None):
                            logged | {value.data})
 
   def call(self, node, func, args, alias_map=None):
-    # TODO(sivachandra): Refactor this method to pass the signature to
-    # simplify.
+    # TODO(b/159052609): We should be passing function signatures to simplify.
     args = args.simplify(node, self.vm)
     self._log_args(arg.bindings for arg in args.posargs)
     ret_map = {}
@@ -1778,7 +1773,14 @@ def call(self, node, func, args, alias_map=None):
       retvar.PasteVariable(result, node)
       all_mutations.update(mutations)
 
-    if all_mutations and self.vm.options.check_container_types:
+    # Don't check container types if the function has multiple bindings.
+    # This is a hack to prevent false positives when we call a method on a
+    # variable with multiple bindings, since we don't always filter rigorously
+    # enough in get_views.
+    # See tests/py3/test_annotations:test_list for an example that would break
+    # if we removed the len(bindings) check.
+    if all_mutations and self.vm.options.check_container_types and (
+        len(func.variable.Bindings(node)) == 1):
       # Raise an error if:
       # - An annotation has a type param that is not ambigious or empty
       # - The mutation adds a type that is not ambiguous or empty
@@ -1794,19 +1796,26 @@ def compatible_with(existing, new):
             return True
         return False
 
+      filtered_mutations = []
       errors = collections.defaultdict(dict)
 
       for obj, name, values in all_mutations:
         if obj.from_annotation:
           params = obj.get_instance_type_parameter(name)
           ps = filter_contents(params)
           if ps:
+            # We filter out mutations to parameters with type Any.
+            filtered_mutations.append((obj, name, values))
             # check if the container type is being broadened.
             vs = filter_contents(values)
             new = [x for x in (vs - ps) if not compatible_with(ps, x)]
             if new:
               formal = name.split(".")[-1]
               errors[obj][formal] = (params, values, obj.from_annotation)
+        else:
+          filtered_mutations.append((obj, name, values))
+
+      all_mutations = filtered_mutations
 
       for obj, errs in errors.items():
         names = {name for _, _, name in errs.values()}
@@ -1888,7 +1897,7 @@ def _call_with_signatures(self, node, func, args, view, signatures):
             view[arg] = arg.AddBinding(self.vm.convert.unsolvable, [], node)
             break
     if self._has_mutable:
-      # TODO(kramm): We only need to whack the type params that appear in
+      # TODO(b/159055015): We only need to whack the type params that appear in
       # a mutable parameter.
       mutations = self._get_mutation_to_unknown(
           node, (view[p].data for p in itertools.chain(
@@ -2876,7 +2885,6 @@ def _map_args(self, node, args):
       kwargs_name = sig.kwargs_name
       # Build a **kwargs dictionary out of the extraneous parameters
       if args.starstarargs:
-        # TODO(kramm): modify type parameters to account for namedargs
         callargs[kwargs_name] = args.starstarargs.AssignToNewVariable(node)
       else:
         omit = sig.param_names + sig.kwonly_params
@@ -3057,20 +3065,16 @@ def _build_signature(self, name, annotations):
         defaults,
         annotations)
 
-  # TODO(kramm): support retrieving the following attributes:
-  # 'func_{code, name, defaults, globals, locals, dict, closure},
-  # '__name__', '__dict__', '__doc__', '_vm', '_func'
-
   def get_first_opcode(self):
     return self.code.co_code[0]
 
   def argcount(self, _):
     return self.code.co_argcount
 
-  def match_args(self, node, args, alias_map):
+  def match_args(self, node, args, alias_map=None, match_all_views=False):
     if not self.signature.has_param_annotations:
       return
-    return super(InterpreterFunction, self).match_args(node, args, alias_map)
+    return super().match_args(node, args, alias_map, match_all_views)
 
   def _inner_cls_check(self, last_frame):
     """Check if the function and its nested class use same type parameter."""
@@ -3174,16 +3178,14 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
     # type-checking down the road.
     annotations = self.vm.annotations_util.sub_annotations(
         node, sig.annotations, substs, instantiate_unbound=False)
-    if annotations:
+    if sig.has_param_annotations:
       for name in callargs:
         if (name in annotations and (not self.is_attribute_of_class or
                                      self.argcount(node) == 0 or
                                      name != sig.param_names[0])):
           extra_key = (self.get_first_opcode(), name)
-          node, callargs[name] = self.vm.init_class(
-              node, annotations[name], extra_key=extra_key)
-          for d in callargs[name].data:
-            d.from_annotation = name
+          node, callargs[name] = self.vm.annotations_util.init_annotation(
+              node, name, annotations[name], extra_key=extra_key)
     try:
       frame = self.vm.make_frame(
           node, self.code, self.f_globals, self.f_locals, callargs,
@@ -3263,7 +3265,14 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
       node2, _ = async_generator.run_generator(node)
       node_after_call, ret = node2, async_generator.to_variable(node2)
     else:
-      node2, ret = self.vm.run_frame(frame, node)
+      if self.vm.options.check_parameter_types:
+        annotated_locals = {
+            name: abstract_utils.Local(node, self.get_first_opcode(), annot,
+                                       callargs.get(name), self.vm)
+            for name, annot in annotations.items() if name != "return"}
+      else:
+        annotated_locals = {}
+      node2, ret = self.vm.run_frame(frame, node, annotated_locals)
       if self.is_coroutine():
         ret = Coroutine(self.vm, ret, node2).to_variable(node2)
       node_after_call = node2
@@ -3544,6 +3553,10 @@ def is_overload(self):
   def is_overload(self, value):
     self.underlying.is_overload = value
 
+  @property
+  def defaults(self):
+    return self.underlying.defaults
+
   def iter_signature_functions(self):
     for f in self.underlying.iter_signature_functions():
       yield self.underlying.bound_class(self._callself, f)
@@ -3942,7 +3955,6 @@ def to_structural_def(self, node, class_name):
     """Convert this Unknown to a pytd.Class."""
     self_param = (pytd.Parameter("self", pytd.AnythingType(),
                                  False, False, None),)
-    # TODO(kramm): Record these.
     starargs = None
     starstarargs = None
     def _make_sig(args, ret):

pytype/abstract_utils.py

@@ -640,3 +640,54 @@ def get_annotations_dict(members):
   except ConversionError:
     return None
   return annots if annots.isinstance_AnnotationsDict() else None
+
+
+class Local:
+  """A possibly annotated local variable."""
+
+  def __init__(self, node, op, typ, orig, vm):
+    self._ops = [op]
+    if typ:
+      self.typ = vm.program.NewVariable([typ], [], node)
+    else:
+      # Creating too many variables bloats the typegraph, hurting performance,
+      # so we use None instead of an empty variable.
+      self.typ = None
+    self.orig = orig
+    self.vm = vm
+
+  @property
+  def last_op(self):
+    # TODO(b/74434237): This property can be removed once the usage of it in
+    # dataclass_overlay is gone.
+    return self._ops[-1]
+
+  @property
+  def stack(self):
+    return self.vm.simple_stack(self.last_op)
+
+  def update(self, node, op, typ, orig):
+    """Update this variable's annotation and/or value."""
+    if op in self._ops:
+      return
+    self._ops.append(op)
+    if typ:
+      if self.typ:
+        self.typ.AddBinding(typ, [], node)
+      else:
+        self.typ = self.vm.program.NewVariable([typ], [], node)
+    if orig:
+      self.orig = orig
+
+  def get_type(self, node, name):
+    """Gets the variable's annotation."""
+    if not self.typ:
+      return None
+    values = self.typ.Data(node)
+    if len(values) > 1:
+      self.vm.errorlog.ambiguous_annotation(self.stack, values, name)
+      return self.vm.convert.unsolvable
+    elif values:
+      return values[0]
+    else:
+      return None

pytype/analyze.py

@@ -87,7 +87,7 @@ def create_kwargs(self, node):
     kwargs.merge_instance_type_parameter(node, abstract_utils.V, value_type)
     return kwargs.to_variable(node)
 
-  def create_method_arguments(self, node, method):
+  def create_method_arguments(self, node, method, use_defaults=False):
     """Create arguments for the given method.
 
     Creates Unknown objects as arguments for the given method. Note that we
@@ -97,14 +97,29 @@ def create_method_arguments(self, node, method):
     Args:
       node: The current node.
       method: An abstract.InterpreterFunction.
+      use_defaults: Whether to use parameter defaults for arguments. When True,
+        unknown arguments are created with force=False, as it is fine to use
+        Unsolvable rather than Unknown objects for type-checking defaults.
 
     Returns:
       A tuple of a node and a function.Args object.
     """
-    args = [self.convert.create_new_unknown(node, force=True)
-            for _ in range(method.argcount(node))]
-    kws = {key: self.convert.create_new_unknown(node, force=True)
-           for key in method.signature.kwonly_params}
+    args = []
+    num_posargs = method.argcount(node)
+    num_posargs_no_default = num_posargs - len(method.defaults)
+    for i in range(num_posargs):
+      default_idx = i - num_posargs_no_default
+      if use_defaults and default_idx >= 0:
+        arg = method.defaults[default_idx]
+      else:
+        arg = self.convert.create_new_unknown(node, force=not use_defaults)
+      args.append(arg)
+    kws = {}
+    for key in method.signature.kwonly_params:
+      if use_defaults and key in method.kw_defaults:
+        kws[key] = method.kw_defaults[key]
+      else:
+        kws[key] = self.convert.create_new_unknown(node, force=not use_defaults)
     starargs = self.create_varargs(node) if method.has_varargs() else None
     starstarargs = self.create_kwargs(node) if method.has_kwargs() else None
     return node, function.Args(posargs=tuple(args),
@@ -332,7 +347,6 @@ def _call_init_on_binding(self, node, b):
 
   def call_init(self, node, instance):
     # Call __init__ on each binding.
-    # TODO(kramm): This should do join_cfg_nodes, instead of concatenating them.
     for b in instance.bindings:
       if b.data in self._initialized_instances:
         continue
@@ -523,7 +537,6 @@ def _call_traces_to_function(call_traces, name_transform=lambda x: x):
           arg_names[i] = function.argname(i)
       arg_types = (a.data.to_type(node) for a in args)
       ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data)
-      # TODO(kramm): Record these:
       starargs = None
       starstarargs = None
       funcs[func.data.name].add(pytd.Signature(
@@ -575,9 +588,6 @@ def pytd_classes_for_call_traces(self):
       ))
     return classes
 
-  def pytd_aliases(self):
-    return ()  # TODO(kramm): Compute these.
-
   def pytd_classes_for_namedtuple_instances(self):
     return tuple(v.generate_ast() for v in self._generated_classes.values())
 
@@ -586,7 +596,7 @@ def compute_types(self, defs):
                tuple(self.pytd_classes_for_call_traces()) +
                self.pytd_classes_for_namedtuple_instances())
     functions = tuple(self.pytd_functions_for_call_traces())
-    aliases = tuple(self.pytd_aliases())
+    aliases = ()  # aliases are instead recorded as constants
     ty = pytd_utils.Concat(
         self.pytd_for_types(defs),
         pytd_utils.CreateModule("unknowns", classes=classes,

pytype/annotations_util.py

@@ -186,6 +186,12 @@ def convert_class_annotations(self, node, raw_annotations):
       annotations[name] = annot or self.vm.convert.unsolvable
     return annotations
 
+  def init_annotation(self, node, name, annot, extra_key=None):
+    node, value = self.vm.init_class(node, annot, extra_key=extra_key)
+    for d in value.data:
+      d.from_annotation = name
+    return node, value
+
   def apply_annotation(self, state, op, name, value):
     """If there is an annotation for the op, return its value."""
     assert op is self.vm.frame.current_opcode
@@ -202,9 +208,7 @@ def apply_annotation(self, state, op, name, value):
           self.vm.frames, annot, details=errorlog.details)
     typ = self.extract_annotation(
         state.node, var, name, self.vm.simple_stack(), is_var=True)
-    _, value = self.vm.init_class(state.node, typ)
-    for d in value.data:
-      d.from_annotation = name
+    _, value = self.init_annotation(state.node, name, typ)
     return typ, value
 
   def extract_annotation(self, node, var, name, stack, is_var=False):

pytype/attribute.py

@@ -271,7 +271,7 @@ def _get_attribute(self, node, obj, cls, name, valself):
           # An attribute has been declared but not defined, e.g.,
           #   class Foo:
           #     bar: int
-          _, attr = self.vm.init_class(node, typ)
+          _, attr = self.vm.annotations_util.init_annotation(node, name, typ)
     if attr is not None:
       attr = self._filter_var(node, attr)
     if attr is None and obj.maybe_missing_members:
@@ -496,8 +496,6 @@ def _set_member(self, node, obj, name, var):
       log.debug("Adding choice(s) to %s: %d new values (%d total)", name,
                 len(variable.bindings) - old_len, len(variable.bindings))
     else:
-      # TODO(kramm): Under what circumstances can we just reuse var?
-      #              (variable = self.members[name] = var)?
       log.debug("Setting %s to the %d values in %r",
                 name, len(var.bindings), var)
       variable = var.AssignToNewVariable(node)

pytype/blocks.py

@@ -285,7 +285,7 @@ def order_code(code):
     A CodeBlocks instance.
   """
   bytecodes = code.co_code
-  add_pop_block_targets(bytecodes)  # TODO(kramm): move into pyc/opcodes.py?
+  add_pop_block_targets(bytecodes)
   return OrderedCode(code, bytecodes, compute_order(bytecodes),
                      code.python_version)
 

pytype/config.py

@@ -148,6 +148,11 @@ def add_basic_options(o):
       "--check-container-types", action="store_true",
       dest="check_container_types", default=False,
       help="Check container mutations against their annotations. " + temporary)
+  o.add_argument(
+      "--check-parameter-types", action="store_true",
+      dest="check_parameter_types", default=False,
+      help=("Check parameter defaults and assignments against their "
+            "annotations. " + temporary))
   o.add_argument(
       "--check-variable-types", action="store_true",
       dest="check_variable_types", default=False,