diff --git a/ChangeLog b/ChangeLog index 50002ac9d..d500329fe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,11 @@ What's New in astroid 2.5.3? ============================ Release Date: TBA +* Takes into account the fact that subscript inferring for a ClassDef may involve __class_getitem__ method + +* Reworks the `collections` and `typing` brain so that `pylint`s acceptance tests are fine. + + Closes PyCQA/pylint#4206 What's New in astroid 2.5.2? ============================ diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index ef6a30a24..031325ea6 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -68,7 +68,7 @@ def __rmul__(self, other): pass""" if PY39: base_deque_class += """ @classmethod - def __class_getitem__(self, item): pass""" + def __class_getitem__(self, item): return cls""" return base_deque_class @@ -77,7 +77,53 @@ def _ordered_dict_mock(): class OrderedDict(dict): def __reversed__(self): return self[::-1] def move_to_end(self, key, last=False): pass""" + if PY39: + base_ordered_dict_class += """ + @classmethod + def __class_getitem__(cls, item): return cls""" return base_ordered_dict_class astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) + + +def _looks_like_subscriptable(node: astroid.nodes.ClassDef) -> bool: + """ + Returns True if the node corresponds to a ClassDef of the Collections.abc module that + supports subscripting + + :param node: ClassDef node + """ + if node.qname().startswith("_collections") or node.qname().startswith( + "collections" + ): + try: + node.getattr("__class_getitem__") + return True + except astroid.AttributeInferenceError: + pass + return False + + +CLASS_GET_ITEM_TEMPLATE = """ +@classmethod +def __class_getitem__(cls, item): + return cls +""" + + +def easy_class_getitem_inference(node, context=None): + # Here __class_getitem__ exists but is quite a mess to infer thus + # put an easy inference tip + func_to_add = astroid.extract_node(CLASS_GET_ITEM_TEMPLATE) + node.locals["__class_getitem__"] = [func_to_add] + + +if PY39: + # Starting with Python39 some objects of the collection module are subscriptable + # thanks to the __class_getitem__ method but the way it is implemented in + # _collection_abc makes it difficult to infer. (We would have to handle AssignName inference in the + # getitem method of the ClassDef class) Instead we put here a mock of the __class_getitem__ method + astroid.MANAGER.register_transform( + astroid.nodes.ClassDef, easy_class_getitem_inference, _looks_like_subscriptable + ) diff --git a/astroid/brain/brain_typing.py b/astroid/brain/brain_typing.py index 3c69bdc24..5fa3c9edf 100644 --- a/astroid/brain/brain_typing.py +++ b/astroid/brain/brain_typing.py @@ -8,7 +8,7 @@ """Astroid hooks for typing.py support.""" import sys import typing -from functools import lru_cache +from functools import partial from astroid import ( MANAGER, @@ -19,6 +19,7 @@ nodes, context, InferenceError, + AttributeInferenceError, ) import astroid @@ -116,37 +117,12 @@ def infer_typedDict( # pylint: disable=invalid-name node.root().locals["TypedDict"] = [class_def] -GET_ITEM_TEMPLATE = """ +CLASS_GETITEM_TEMPLATE = """ @classmethod -def __getitem__(cls, value): +def __class_getitem__(cls, item): return cls """ -ABC_METACLASS_TEMPLATE = """ -from abc import ABCMeta -ABCMeta -""" - - -@lru_cache() -def create_typing_metaclass(): - #  Needs to mock the __getitem__ class method so that - #  MutableSet[T] is acceptable - func_to_add = extract_node(GET_ITEM_TEMPLATE) - - abc_meta = next(extract_node(ABC_METACLASS_TEMPLATE).infer()) - typing_meta = nodes.ClassDef( - name="ABCMeta_typing", - lineno=abc_meta.lineno, - col_offset=abc_meta.col_offset, - parent=abc_meta.parent, - ) - typing_meta.postinit( - bases=[extract_node(ABC_METACLASS_TEMPLATE)], body=[], decorators=None - ) - typing_meta.locals["__getitem__"] = [func_to_add] - return typing_meta - def _looks_like_typing_alias(node: nodes.Call) -> bool: """ @@ -161,10 +137,43 @@ def _looks_like_typing_alias(node: nodes.Call) -> bool: isinstance(node, nodes.Call) and isinstance(node.func, nodes.Name) and node.func.name == "_alias" - and isinstance(node.args[0], nodes.Attribute) + and ( + # _alias function works also for builtins object such as list and dict + isinstance(node.args[0], nodes.Attribute) + or isinstance(node.args[0], nodes.Name) + and node.args[0].name != "type" + ) ) +def _forbid_class_getitem_access(node: nodes.ClassDef) -> None: + """ + Disable the access to __class_getitem__ method for the node in parameters + """ + + def full_raiser(origin_func, attr, *args, **kwargs): + """ + Raises an AttributeInferenceError in case of access to __class_getitem__ method. + Otherwise just call origin_func. + """ + if attr == "__class_getitem__": + raise AttributeInferenceError("__class_getitem__ access is not allowed") + else: + return origin_func(attr, *args, **kwargs) + + if not isinstance(node, nodes.ClassDef): + raise TypeError("The parameter type should be ClassDef") + try: + node.getattr("__class_getitem__") + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + # protocol defined in collections module) whereas the typing module consider it should not + # We do not want __class_getitem__ to be found in the classdef + partial_raiser = partial(full_raiser, node.getattr) + node.getattr = partial_raiser + except AttributeInferenceError: + pass + + def infer_typing_alias( node: nodes.Call, ctx: context.InferenceContext = None ) -> typing.Optional[node_classes.NodeNG]: @@ -174,38 +183,48 @@ def infer_typing_alias( :param node: call node :param context: inference context """ - if not isinstance(node, nodes.Call): - return None res = next(node.args[0].infer(context=ctx)) if res != astroid.Uninferable and isinstance(res, nodes.ClassDef): - class_def = nodes.ClassDef( - name=f"{res.name}_typing", - lineno=0, - col_offset=0, - parent=res.parent, - ) - class_def.postinit( - bases=[res], - body=res.body, - decorators=res.decorators, - metaclass=create_typing_metaclass(), - ) - return class_def - - if len(node.args) == 2 and isinstance(node.args[0], nodes.Attribute): - class_def = nodes.ClassDef( - name=node.args[0].attrname, - lineno=0, - col_offset=0, - parent=node.parent, - ) - class_def.postinit( - bases=[], body=[], decorators=None, metaclass=create_typing_metaclass() - ) - return class_def - - return None + if not PY39: + # Here the node is a typing object which is an alias toward + # the corresponding object of collection.abc module. + # Before python3.9 there is no subscript allowed for any of the collections.abc objects. + # The subscript ability is given through the typing._GenericAlias class + # which is the metaclass of the typing object but not the metaclass of the inferred + # collections.abc object. + # Thus we fake subscript ability of the collections.abc object + # by mocking the existence of a __class_getitem__ method. + # We can not add `__getitem__` method in the metaclass of the object because + # the metaclass is shared by subscriptable and not subscriptable object + maybe_type_var = node.args[1] + if not ( + isinstance(maybe_type_var, node_classes.Tuple) + and not maybe_type_var.elts + ): + # The typing object is subscriptable if the second argument of the _alias function + # is a TypeVar or a tuple of TypeVar. We could check the type of the second argument but + # it appears that in the typing module the second argument is only TypeVar or a tuple of TypeVar or empty tuple. + # This last value means the type is not Generic and thus cannot be subscriptable + func_to_add = astroid.extract_node(CLASS_GETITEM_TEMPLATE) + res.locals["__class_getitem__"] = [func_to_add] + else: + # If we are here, then we are sure to modify object that do have __class_getitem__ method (which origin is one the + # protocol defined in collections module) whereas the typing module consider it should not + # We do not want __class_getitem__ to be found in the classdef + _forbid_class_getitem_access(res) + else: + # Within python3.9 discrepencies exist between some collections.abc containers that are subscriptable whereas + # corresponding containers in the typing module are not! This is the case at least for ByteString. + # It is far more to complex and dangerous to try to remove __class_getitem__ method from all the ancestors of the + # current class. Instead we raise an AttributeInferenceError if we try to access it. + maybe_type_var = node.args[1] + if isinstance(maybe_type_var, nodes.Const) and maybe_type_var.value == 0: + # Starting with Python39 the _alias function is in fact instantiation of _SpecialGenericAlias class. + # Thus the type is not Generic if the second argument of the call is equal to zero + _forbid_class_getitem_access(res) + return iter([res]) + return iter([astroid.Uninferable]) MANAGER.register_transform( @@ -223,4 +242,6 @@ def infer_typing_alias( ) if PY37: - MANAGER.register_transform(nodes.Call, infer_typing_alias, _looks_like_typing_alias) + MANAGER.register_transform( + nodes.Call, inference_tip(infer_typing_alias), _looks_like_typing_alias + ) diff --git a/astroid/scoped_nodes.py b/astroid/scoped_nodes.py index 74bc97923..dd5aa1257 100644 --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -54,6 +54,8 @@ from astroid import util +PY39 = sys.version_info[:2] >= (3, 9) + BUILTINS = builtins.__name__ ITER_METHODS = ("__iter__", "__getitem__") EXCEPTION_BASE_CLASSES = frozenset({"Exception", "BaseException"}) @@ -2617,7 +2619,22 @@ def getitem(self, index, context=None): try: methods = dunder_lookup.lookup(self, "__getitem__") except exceptions.AttributeInferenceError as exc: - raise exceptions.AstroidTypeError(node=self, context=context) from exc + if isinstance(self, ClassDef): + # subscripting a class definition may be + # achieved thanks to __class_getitem__ method + # which is a classmethod defined in the class + # that supports subscript and not in the metaclass + try: + methods = self.getattr("__class_getitem__") + # Here it is assumed that the __class_getitem__ node is + # a FunctionDef. One possible improvement would be to deal + # with more generic inference. + except exceptions.AttributeInferenceError: + raise exceptions.AstroidTypeError( + node=self, context=context + ) from exc + else: + raise exceptions.AstroidTypeError(node=self, context=context) from exc method = methods[0] @@ -2627,6 +2644,19 @@ def getitem(self, index, context=None): try: return next(method.infer_call_result(self, new_context)) + except AttributeError: + # Starting with python3.9, builtin types list, dict etc... + # are subscriptable thanks to __class_getitem___ classmethod. + # However in such case the method is bound to an EmptyNode and + # EmptyNode doesn't have infer_call_result method yielding to + # AttributeError + if ( + isinstance(method, node_classes.EmptyNode) + and self.name in ("list", "dict", "set", "tuple", "frozenset") + and PY39 + ): + return self + raise except exceptions.InferenceError: return util.Uninferable diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index fe3ba2489..57b9dc091 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -1026,6 +1026,174 @@ def test_invalid_type_subscript(self): with self.assertRaises(astroid.exceptions.AttributeInferenceError): meth_inf = val_inf.getattr("__class_getitem__")[0] + @test_utils.require_version(minver="3.9") + def test_builtin_subscriptable(self): + """ + Starting with python3.9 builtin type such as list are subscriptable + """ + for typename in ("tuple", "list", "dict", "set", "frozenset"): + src = """ + {:s}[int] + """.format( + typename + ) + right_node = builder.extract_node(src) + inferred = next(right_node.infer()) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], nodes.FunctionDef) + + +def check_metaclass_is_abc(node: nodes.ClassDef): + meta = node.metaclass() + assert isinstance(meta, nodes.ClassDef) + assert meta.name == "ABCMeta" + + +class CollectionsBrain(unittest.TestCase): + def test_collections_object_not_subscriptable(self): + """ + Test that unsubscriptable types are detected + Hashable is not subscriptable even with python39 + """ + wrong_node = builder.extract_node( + """ + import collections.abc + collections.abc.Hashable[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + collections.abc.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Hashable", + "object", + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr("__class_getitem__") + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable(self): + """Starting with python39 some object of collections module are subscriptable. Test one of them""" + right_node = builder.extract_node( + """ + import collections.abc + collections.abc.MutableSet[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object", + ], + ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + + @test_utils.require_version(maxver="3.9") + def test_collections_object_not_yet_subscriptable(self): + """ + Test that unsubscriptable types are detected as such. + Until python39 MutableSet of the collections module is not subscriptable. + """ + wrong_node = builder.extract_node( + """ + import collections.abc + collections.abc.MutableSet[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import collections.abc + collections.abc.MutableSet + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object", + ], + ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr("__class_getitem__") + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable_2(self): + """Starting with python39 Iterator in the collection.abc module is subscriptable""" + node = builder.extract_node( + """ + import collections.abc + class Derived(collections.abc.Iterator[int]): + pass + """ + ) + inferred = next(node.infer()) + check_metaclass_is_abc(inferred) + assertEqualMro( + inferred, + [ + "Derived", + "Iterator", + "Iterable", + "object", + ], + ) + + @test_utils.require_version(maxver="3.8") + def test_collections_object_not_yet_subscriptable_2(self): + """Before python39 Iterator in the collection.abc module is not subscriptable""" + node = builder.extract_node( + """ + import collections.abc + collections.abc.Iterator[int] + """ + ) + with self.assertRaises(astroid.exceptions.InferenceError): + next(node.infer()) + + @test_utils.require_version(minver="3.9") + def test_collections_object_subscriptable_3(self): + """With python39 ByteString class of the colletions module is subscritable (but not the same class from typing module)""" + right_node = builder.extract_node( + """ + import collections.abc + collections.abc.ByteString[int] + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + @test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): @@ -1211,25 +1379,13 @@ class CustomTD(TypedDict): assert len(typing_module.locals["TypedDict"]) == 1 assert inferred_base == typing_module.locals["TypedDict"][0] - @test_utils.require_version("3.8") + @test_utils.require_version(minver="3.7") def test_typing_alias_type(self): """ Test that the type aliased thanks to typing._alias function are correctly inferred. + typing_alias function is introduced with python37 """ - - def check_metaclass(node: nodes.ClassDef): - meta = node.metaclass() - assert isinstance(meta, nodes.ClassDef) - assert meta.name == "ABCMeta_typing" - assert "ABCMeta" == meta.basenames[0] - assert meta.locals.get("__getitem__") is not None - - abc_meta = next(meta.bases[0].infer()) - assert isinstance(abc_meta, nodes.ClassDef) - assert abc_meta.name == "ABCMeta" - assert abc_meta.locals.get("__getitem__") is None - node = builder.extract_node( """ from typing import TypeVar, MutableSet @@ -1242,12 +1398,11 @@ class Derived1(MutableSet[T]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ "Derived1", - "MutableSet_typing", "MutableSet", "Set", "Collection", @@ -1258,6 +1413,14 @@ class Derived1(MutableSet[T]): ], ) + @test_utils.require_version(minver="3.7.2") + def test_typing_alias_type_2(self): + """ + Test that the type aliased thanks to typing._alias function are + correctly inferred. + typing_alias function is introduced with python37. + OrderedDict in the typing module appears only with python 3.7.2 + """ node = builder.extract_node( """ import typing @@ -1266,60 +1429,107 @@ class Derived2(typing.OrderedDict[int, str]): """ ) inferred = next(node.infer()) - check_metaclass(inferred) + # OrderedDict has no metaclass because it + # inherits from dict which is C coded + self.assertIsNone(inferred.metaclass()) assertEqualMro( inferred, [ "Derived2", - "OrderedDict_typing", "OrderedDict", "dict", "object", ], ) - node = builder.extract_node( + def test_typing_object_not_subscriptable(self): + """Hashable is not subscriptable""" + wrong_node = builder.extract_node( """ import typing - class Derived3(typing.Pattern[str]): - pass + typing.Hashable[int] """ ) - inferred = next(node.infer()) - check_metaclass(inferred) + with self.assertRaises(astroid.exceptions.InferenceError): + next(wrong_node.infer()) + right_node = builder.extract_node( + """ + import typing + typing.Hashable + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ - "Derived3", - "Pattern", + "Hashable", "object", ], ) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr("__class_getitem__") - @test_utils.require_version("3.8") - def test_typing_alias_side_effects(self): - """Test that typing._alias changes doesn't have unwanted consequences.""" - node = builder.extract_node( + @test_utils.require_version(minver="3.7") + def test_typing_object_subscriptable(self): + """Test that MutableSet is subscriptable""" + right_node = builder.extract_node( """ import typing - import collections.abc - - class Derived(collections.abc.Iterator[int]): - pass + typing.MutableSet[int] """ ) - inferred = next(node.infer()) - assert inferred.metaclass() is None # Should this be ABCMeta? + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) assertEqualMro( inferred, [ - "Derived", - # Should this be more? - # "Iterator_typing"? - # "Iterator", - # "object", + "MutableSet", + "Set", + "Collection", + "Sized", + "Iterable", + "Container", + "object", ], ) + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + + @test_utils.require_version(minver="3.7") + def test_typing_object_notsubscriptable_3(self): + """Until python39 ByteString class of the typing module is not subscritable (whereas it is in the collections module)""" + right_node = builder.extract_node( + """ + import typing + typing.ByteString + """ + ) + inferred = next(right_node.infer()) + check_metaclass_is_abc(inferred) + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + self.assertIsInstance( + inferred.getattr("__class_getitem__")[0], nodes.FunctionDef + ) + + @test_utils.require_version(minver="3.9") + def test_typing_object_builtin_subscriptable(self): + """ + Test that builtins alias, such as typing.List, are subscriptable + """ + # Do not test Tuple as it is inferred as _TupleType class (needs a brain?) + for typename in ("List", "Dict", "Set", "FrozenSet"): + src = """ + import typing + typing.{:s}[int] + """.format( + typename + ) + right_node = builder.extract_node(src) + inferred = next(right_node.infer()) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertIsInstance(inferred.getattr("__iter__")[0], nodes.FunctionDef) class ReBrainTest(unittest.TestCase):