From d081c0d02e686f81b687c2dbb0b0c652aba86b04 Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Fri, 22 Jan 2021 11:44:31 +0100 Subject: [PATCH 1/3] Fix misdetection of some 3.6 typing[_extensions] Fixes #403 --- cloudpickle/cloudpickle.py | 15 +++++++++++---- tests/cloudpickle_test.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 05d52afa0..9bf57bea9 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -452,15 +452,22 @@ def _extract_class_dict(cls): if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): - # This is very cheap but might generate false positives. + # This is very cheap but might generate false positives. So try to narrow it down is good as possible + type_module = getattr(type(obj), '__module__', None) + from_typing_extensions = type_module == 'typing_extensions' + from_typing = type_module == 'typing' + # general typing Constructs is_typing = getattr(obj, '__origin__', None) is not None # typing_extensions.Literal - is_litteral = getattr(obj, '__values__', None) is not None + is_litteral = (getattr(obj, '__values__', None) is not None) and from_typing_extensions # typing_extensions.Final - is_final = getattr(obj, '__type__', None) is not None + is_final = (getattr(obj, '__type__', None) is not None) and from_typing_extensions + + # typing.ClassVar + is_classvar = (getattr(obj, '__type__', None) is not None) and from_typing # typing.Union/Tuple for old Python 3.5 is_union = getattr(obj, '__union_params__', None) is not None @@ -469,7 +476,7 @@ def _is_parametrized_type_hint(obj): getattr(obj, '__result__', None) is not None and getattr(obj, '__args__', None) is not None ) - return any((is_typing, is_litteral, is_final, is_union, is_tuple, + return any((is_typing, is_litteral, is_final, is_classvar, is_union, is_tuple, is_callable)) def _create_parametrized_type_hint(origin, args): diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index a456b6372..4ada7a552 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2283,6 +2283,26 @@ def reduce_myclass(x): finally: copyreg.dispatch_table.pop(MyClass) + def test_literal_misdetection(self): + # see https://github.com/cloudpipe/cloudpickle/issues/403 + class MyClass: + @property + def __values__(self): + return () + + o = MyClass() + pickle_depickle(o, protocol=self.protocol) + + def test_final_or_classvar_misdetection(self): + # see https://github.com/cloudpipe/cloudpickle/issues/403 + class MyClass: + @property + def __type__(self): + return int + + o = MyClass() + pickle_depickle(o, protocol=self.protocol) + class Protocol2CloudPickleTest(CloudPickleTest): From a4b48aa457b5dbe931f7ced791f5a10816866f5d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 1 Mar 2021 17:54:51 +0100 Subject: [PATCH 2/3] Apply suggestions from code review PEP8 --- cloudpickle/cloudpickle.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index d076b7fdc..20e9a9550 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -455,7 +455,8 @@ def _extract_class_dict(cls): if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): - # This is very cheap but might generate false positives. So try to narrow it down is good as possible + # This is very cheap but might generate false positives. So try to + # narrow it down is good as possible. type_module = getattr(type(obj), '__module__', None) from_typing_extensions = type_module == 'typing_extensions' from_typing = type_module == 'typing' @@ -476,7 +477,9 @@ def _is_parametrized_type_hint(obj): ) # typing.ClassVar - is_classvar = (getattr(obj, '__type__', None) is not None) and from_typing + is_classvar = ( + (getattr(obj, '__type__', None) is not None) and from_typing + ) # typing.Union/Tuple for old Python 3.5 is_union = getattr(obj, '__union_params__', None) is not None From cfd4a36fbff2c43dd357438760431ac9b8f31251 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 1 Mar 2021 18:02:17 +0100 Subject: [PATCH 3/3] Add changelog entry --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 4eda950fd..0270c0386 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,10 @@ dev https://www.python.org/dev/peps/pep-0563/ ([PR #400](https://github.com/cloudpipe/cloudpickle/pull/400)) +- Stricter parametrized type detection heuristics in + _is_parametrized_type_hint to limit false positives. + ([PR #409](https://github.com/cloudpipe/cloudpickle/pull/409)) + 1.6.0 =====