From e61da59eadd00dc45464c6ec8c1c916b20e6c40a Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Mon, 20 Dec 2021 11:46:12 +0800 Subject: [PATCH 1/6] Fix import of pickle module on PYPY 3.8 (#455) --- .github/workflows/testing.yml | 6 +++--- CHANGES.md | 3 +++ cloudpickle/cloudpickle.py | 5 +---- cloudpickle/cloudpickle_fast.py | 4 ++-- cloudpickle/compat.py | 4 +++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d51a49096..a5d4ee859 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: [3.6, 3.7, 3.8, 3.9, "3.10-dev", "pypy3"] + python_version: [3.6, 3.7, 3.8, 3.9, "3.10-dev", "pypy-3.8"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions @@ -41,7 +41,7 @@ jobs: python_version: 3.8 # as of 4/02/2020, psutil won't build under PyPy + Windows - os: windows-latest - python_version: "pypy3" + python_version: "pypy-3.8" - os: macos-latest python_version: 3.6 - os: macos-latest @@ -51,7 +51,7 @@ jobs: - os: macos-latest # numpy triggers: RuntimeError: Polyfit sanity test emitted a # warning - python_version: "pypy3" + python_version: "pypy-3.8" runs-on: ${{ matrix.os }} diff --git a/CHANGES.md b/CHANGES.md index 972cf3f07..1d55fdecd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,9 @@ and `abc.abstractstaticmethod`. ([PR #450](https://github.com/cloudpipe/cloudpickle/pull/450)) +- Fix import of pickle module on PYPY 3.8. + ([issue #455](https://github.com/cloudpipe/cloudpickle/issues/455)) + 2.0.0 ===== diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 80ccdea68..e0fcf0668 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -44,7 +44,6 @@ import builtins import dis import opcode -import platform import sys import types import weakref @@ -53,7 +52,7 @@ import typing import warnings -from .compat import pickle +from .compat import pickle, PYPY from collections import OrderedDict from typing import ClassVar, Generic, Union, Tuple, Callable from pickle import _getattribute @@ -92,8 +91,6 @@ def g(): _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() -PYPY = platform.python_implementation() == "PyPy" - builtin_code_type = None if PYPY: # builtin-code objects only exist in pypy diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 70b5ecece..a7b9d194a 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -25,14 +25,14 @@ from enum import Enum from collections import ChainMap, OrderedDict -from .compat import pickle, Pickler +from .compat import pickle, Pickler, PYPY from .cloudpickle import ( _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, - _is_parametrized_type_hint, PYPY, cell_set, + _is_parametrized_type_hint, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, _make_dict_keys, _make_dict_values, _make_dict_items, diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index afa285f62..f26f63266 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,7 +1,9 @@ import sys +import platform +PYPY = platform.python_implementation() == "PyPy" -if sys.version_info < (3, 8): +if sys.version_info < (3, 8) or PYPY: try: import pickle5 as pickle # noqa: F401 from pickle5 import Pickler # noqa: F401 From c26cd3b4535632cbcea495c0effbdbe4878ed340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Wed, 7 Sep 2022 16:55:18 +0200 Subject: [PATCH 2/6] Fix for PyPy > 3.7 --- cloudpickle/cloudpickle_fast.py | 50 ++++++++++++++++++--------------- cloudpickle/compat.py | 11 ++++---- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index e1675925e..05881aff4 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -39,7 +39,7 @@ ) -if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: +if pickle.HIGHEST_PROTOCOL >= 5: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): @@ -642,6 +642,33 @@ def dump(self, obj): raise if pickle.HIGHEST_PROTOCOL >= 5: + def __init__(self, file, protocol=None, buffer_callback=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__( + self, file, protocol=protocol, buffer_callback=buffer_callback + ) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + self.proto = int(protocol) + else: + def __init__(self, file, protocol=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__(self, file, protocol=protocol) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + assert hasattr(self, 'proto') + + + if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: + # Pickler is the C implementation of the CPython pickler and therefore + # we rely on reduce_override method to customize the pickler behavior. + # `CloudPickler.dispatch` is only left for backward compatibility - note # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler @@ -662,17 +689,6 @@ def dump(self, obj): # availability of both notions coincide on CPython's pickle and the # pickle5 backport, but it may not be the case anymore when pypy # implements protocol 5 - def __init__(self, file, protocol=None, buffer_callback=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__( - self, file, protocol=protocol, buffer_callback=buffer_callback - ) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - self.proto = int(protocol) def reducer_override(self, obj): """Type-agnostic reducing callback for function and classes. @@ -733,16 +749,6 @@ def reducer_override(self, obj): # hard-coded call to save_global when pickling meta-classes. dispatch = Pickler.dispatch.copy() - def __init__(self, file, protocol=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - assert hasattr(self, 'proto') - def _save_reduce_pickle5(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None): save = self.save diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index b0f580950..c8fa8a2e0 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,15 +1,16 @@ import sys -import platform - -PYPY = platform.python_implementation() == "PyPy" - -if sys.version_info < (3, 8) or PYPY: +if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 from pickle5 import Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 + + # Use the Python pickler for old CPython versions from pickle import _Pickler as Pickler # noqa: F401 else: import pickle # noqa: F401 + + # Pickler will the C implementation in CPython and the Python + # implementation in PyPy from pickle import Pickler # noqa: F401 From 6b09ea707b6a8de82d4a46080bceca9c3cfaf705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Wed, 7 Sep 2022 16:56:30 +0200 Subject: [PATCH 3/6] Use PyPy 3.9 in the CI --- .github/workflows/testing.yml | 6 +++--- cloudpickle/compat.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index fde554516..bf79a7e1c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy-3.8"] + python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy-3.9"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions @@ -41,7 +41,7 @@ jobs: python_version: 3.8 # as of 4/02/2020, psutil won't build under PyPy + Windows - os: windows-latest - python_version: "pypy-3.8" + python_version: "pypy-3.9" - os: macos-latest python_version: 3.6 - os: macos-latest @@ -51,7 +51,7 @@ jobs: - os: macos-latest # numpy triggers: RuntimeError: Polyfit sanity test emitted a # warning - python_version: "pypy-3.8" + python_version: "pypy-3.9" runs-on: ${{ matrix.os }} diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index c8fa8a2e0..5e9b52773 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,4 +1,6 @@ import sys + + if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 From efe102853b4d5df4cc12385e7bee41ed9419d947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Wed, 7 Sep 2022 17:12:48 +0200 Subject: [PATCH 4/6] lint --- cloudpickle/cloudpickle.py | 5 ++++- cloudpickle/cloudpickle_fast.py | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index a366a6787..317be6915 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -44,6 +44,7 @@ import builtins import dis import opcode +import platform import sys import types import weakref @@ -52,7 +53,7 @@ import typing import warnings -from .compat import pickle, PYPY +from .compat import pickle from collections import OrderedDict from typing import ClassVar, Generic, Union, Tuple, Callable from pickle import _getattribute @@ -91,6 +92,8 @@ def g(): _DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() _DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() +PYPY = platform.python_implementation() == "PyPy" + builtin_code_type = None if PYPY: # builtin-code objects only exist in pypy diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 05881aff4..8741dcbda 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -25,14 +25,14 @@ from enum import Enum from collections import ChainMap, OrderedDict -from .compat import pickle, Pickler, PYPY +from .compat import pickle, Pickler from .cloudpickle import ( _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, - _is_parametrized_type_hint, cell_set, + _is_parametrized_type_hint, PYPY, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, _make_dict_keys, _make_dict_values, _make_dict_items, _make_function, @@ -566,7 +566,6 @@ class CloudPickler(Pickler): _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce _dispatch_table[abc.abstractproperty] = _property_reduce - dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) # function reducers are defined as instance methods of CloudPickler @@ -664,7 +663,6 @@ def __init__(self, file, protocol=None): self.globals_ref = {} assert hasattr(self, 'proto') - if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: # Pickler is the C implementation of the CPython pickler and therefore # we rely on reduce_override method to customize the pickler behavior. From ffaac42911b2c6f14db90e87aae15e273f7cf4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Wed, 7 Sep 2022 17:50:41 +0200 Subject: [PATCH 5/6] Test old CPython version without pickle5 installed --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d1cf1be14..33d83f0b9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5 >=0.0.11 ; python_version <= '3.7' and python_implementation == 'CPython' +pickle5 >=0.0.11 ; python_version == '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From 60bd181113d5f6aac44b89ea6b537c063dc5bd13 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 7 Sep 2022 17:58:49 +0200 Subject: [PATCH 6/6] More precise changelog entry. --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 585d7c2fb..11708bd54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ 2.2.0 (in development) ====================== -- Fix import of pickle module on PYPY 3.8. +- Fix support of PyPy 3.8 and later. ([issue #455](https://github.com/cloudpipe/cloudpickle/issues/455)) 2.1.0