FIX: Support PyPy > 3.7 (#480)

Co-authored-by: Eduardo Schettino <schettino72@gmail.com>
Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>

.github/workflows/testing.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy3"]
+        python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy-3.9"]
         exclude:
           # Do not test all minor versions on all platforms, especially if they
           # are not the oldest/newest supported versions
@@ -41,7 +41,7 @@ jobs:
             python_version: 3.8
             # as of  4/02/2020, psutil won't build under PyPy + Windows
           - os: windows-latest
-            python_version: "pypy3"
+            python_version: "pypy-3.9"
           - os: macos-latest
             python_version: 3.6
           - os: macos-latest
@@ -51,7 +51,7 @@ jobs:
           - os: macos-latest
             # numpy triggers: RuntimeError: Polyfit sanity test emitted a
             # warning
-            python_version: "pypy3"
+            python_version: "pypy-3.9"
 
     runs-on: ${{ matrix.os }}
 

CHANGES.md

@@ -1,7 +1,8 @@
 2.2.0 (in development)
 ======================
 
-- TODO document changes here.
+- Fix support of PyPy 3.8 and later.
+  ([issue #455](https://github.com/cloudpipe/cloudpickle/issues/455))
 
 2.1.0
 =====

cloudpickle/cloudpickle_fast.py

@@ -39,7 +39,7 @@
 )
 
 
-if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY:
+if pickle.HIGHEST_PROTOCOL >= 5:
     # Shorthands similar to pickle.dump/pickle.dumps
 
     def dump(obj, file, protocol=None, buffer_callback=None):
@@ -566,7 +566,6 @@ class CloudPickler(Pickler):
     _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce
     _dispatch_table[abc.abstractproperty] = _property_reduce
 
-
     dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table)
 
     # function reducers are defined as instance methods of CloudPickler
@@ -642,6 +641,32 @@ def dump(self, obj):
                 raise
 
     if pickle.HIGHEST_PROTOCOL >= 5:
+        def __init__(self, file, protocol=None, buffer_callback=None):
+            if protocol is None:
+                protocol = DEFAULT_PROTOCOL
+            Pickler.__init__(
+                self, file, protocol=protocol, buffer_callback=buffer_callback
+            )
+            # map functions __globals__ attribute ids, to ensure that functions
+            # sharing the same global namespace at pickling time also share
+            # their global namespace at unpickling time.
+            self.globals_ref = {}
+            self.proto = int(protocol)
+    else:
+        def __init__(self, file, protocol=None):
+            if protocol is None:
+                protocol = DEFAULT_PROTOCOL
+            Pickler.__init__(self, file, protocol=protocol)
+            # map functions __globals__ attribute ids, to ensure that functions
+            # sharing the same global namespace at pickling time also share
+            # their global namespace at unpickling time.
+            self.globals_ref = {}
+            assert hasattr(self, 'proto')
+
+    if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY:
+        # Pickler is the C implementation of the CPython pickler and therefore
+        # we rely on reduce_override method to customize the pickler behavior.
+
         # `CloudPickler.dispatch` is only left for backward compatibility - note
         # that when using protocol 5, `CloudPickler.dispatch` is not an
         # extension of `Pickler.dispatch` dictionary, because CloudPickler
@@ -662,17 +687,6 @@ def dump(self, obj):
         # availability of both notions coincide on CPython's pickle and the
         # pickle5 backport, but it may not be the case anymore when pypy
         # implements protocol 5
-        def __init__(self, file, protocol=None, buffer_callback=None):
-            if protocol is None:
-                protocol = DEFAULT_PROTOCOL
-            Pickler.__init__(
-                self, file, protocol=protocol, buffer_callback=buffer_callback
-            )
-            # map functions __globals__ attribute ids, to ensure that functions
-            # sharing the same global namespace at pickling time also share
-            # their global namespace at unpickling time.
-            self.globals_ref = {}
-            self.proto = int(protocol)
 
         def reducer_override(self, obj):
             """Type-agnostic reducing callback for function and classes.
@@ -733,16 +747,6 @@ def reducer_override(self, obj):
         # hard-coded call to save_global when pickling meta-classes.
         dispatch = Pickler.dispatch.copy()
 
-        def __init__(self, file, protocol=None):
-            if protocol is None:
-                protocol = DEFAULT_PROTOCOL
-            Pickler.__init__(self, file, protocol=protocol)
-            # map functions __globals__ attribute ids, to ensure that functions
-            # sharing the same global namespace at pickling time also share
-            # their global namespace at unpickling time.
-            self.globals_ref = {}
-            assert hasattr(self, 'proto')
-
         def _save_reduce_pickle5(self, func, args, state=None, listitems=None,
                                  dictitems=None, state_setter=None, obj=None):
             save = self.save

cloudpickle/compat.py

@@ -7,7 +7,12 @@
         from pickle5 import Pickler  # noqa: F401
     except ImportError:
         import pickle  # noqa: F401
+
+        # Use the Python pickler for old CPython versions
         from pickle import _Pickler as Pickler  # noqa: F401
 else:
     import pickle  # noqa: F401
+
+    # Pickler will the C implementation in CPython and the Python
+    # implementation in PyPy
     from pickle import Pickler  # noqa: F401

dev-requirements.txt

@@ -4,7 +4,7 @@ pytest
 pytest-cov
 psutil
 # To test on older Python versions
-pickle5 >=0.0.11 ; python_version <= '3.7' and python_implementation == 'CPython'
+pickle5 >=0.0.11 ; python_version == '3.7' and python_implementation == 'CPython'
 # To be able to test tornado coroutines
 tornado
 # To be able to test numpy specific things