From 23cbe1553c6e8e5662336ced2ec42a66fc3b812c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Wed, 7 Sep 2022 18:02:07 +0200 Subject: [PATCH] FIX: Support PyPy > 3.7 (#480) Co-authored-by: Eduardo Schettino Co-authored-by: Olivier Grisel --- .github/workflows/testing.yml | 6 ++-- CHANGES.md | 3 +- cloudpickle/cloudpickle_fast.py | 50 ++++++++++++++++++--------------- cloudpickle/compat.py | 5 ++++ dev-requirements.txt | 2 +- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c60f0d2da..bf79a7e1c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy3"] + python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy-3.9"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions @@ -41,7 +41,7 @@ jobs: python_version: 3.8 # as of 4/02/2020, psutil won't build under PyPy + Windows - os: windows-latest - python_version: "pypy3" + python_version: "pypy-3.9" - os: macos-latest python_version: 3.6 - os: macos-latest @@ -51,7 +51,7 @@ jobs: - os: macos-latest # numpy triggers: RuntimeError: Polyfit sanity test emitted a # warning - python_version: "pypy3" + python_version: "pypy-3.9" runs-on: ${{ matrix.os }} diff --git a/CHANGES.md b/CHANGES.md index e1fc4a1d7..11708bd54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,8 @@ 2.2.0 (in development) ====================== -- TODO document changes here. +- Fix support of PyPy 3.8 and later. + ([issue #455](https://github.com/cloudpipe/cloudpickle/issues/455)) 2.1.0 ===== diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index a0fd44a1c..8741dcbda 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -39,7 +39,7 @@ ) -if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: +if pickle.HIGHEST_PROTOCOL >= 5: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): @@ -566,7 +566,6 @@ class CloudPickler(Pickler): _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce _dispatch_table[abc.abstractproperty] = _property_reduce - dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) # function reducers are defined as instance methods of CloudPickler @@ -642,6 +641,32 @@ def dump(self, obj): raise if pickle.HIGHEST_PROTOCOL >= 5: + def __init__(self, file, protocol=None, buffer_callback=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__( + self, file, protocol=protocol, buffer_callback=buffer_callback + ) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + self.proto = int(protocol) + else: + def __init__(self, file, protocol=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__(self, file, protocol=protocol) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + assert hasattr(self, 'proto') + + if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: + # Pickler is the C implementation of the CPython pickler and therefore + # we rely on reduce_override method to customize the pickler behavior. + # `CloudPickler.dispatch` is only left for backward compatibility - note # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler @@ -662,17 +687,6 @@ def dump(self, obj): # availability of both notions coincide on CPython's pickle and the # pickle5 backport, but it may not be the case anymore when pypy # implements protocol 5 - def __init__(self, file, protocol=None, buffer_callback=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__( - self, file, protocol=protocol, buffer_callback=buffer_callback - ) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - self.proto = int(protocol) def reducer_override(self, obj): """Type-agnostic reducing callback for function and classes. @@ -733,16 +747,6 @@ def reducer_override(self, obj): # hard-coded call to save_global when pickling meta-classes. dispatch = Pickler.dispatch.copy() - def __init__(self, file, protocol=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - assert hasattr(self, 'proto') - def _save_reduce_pickle5(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None): save = self.save diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index b37e3da1a..5e9b52773 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -7,7 +7,12 @@ from pickle5 import Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 + + # Use the Python pickler for old CPython versions from pickle import _Pickler as Pickler # noqa: F401 else: import pickle # noqa: F401 + + # Pickler will the C implementation in CPython and the Python + # implementation in PyPy from pickle import Pickler # noqa: F401 diff --git a/dev-requirements.txt b/dev-requirements.txt index d1cf1be14..33d83f0b9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5 >=0.0.11 ; python_version <= '3.7' and python_implementation == 'CPython' +pickle5 >=0.0.11 ; python_version == '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things