From 0cb191032d4b7913e6a66e3f24788668efbd10dd Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 18 May 2022 14:25:32 -0700 Subject: [PATCH 1/6] Improve compatibility with "nogil" Python and 3.11 This makes a number of changes to improve compatibility with the "nogil" Python fork as well as the upcoming 3.11 release. - Fix _code_reduce for 3.11b0 and nogil Python - Use instr.argval in _walk_global_ops. This avoids adding a special case for 3.11+ (and is useful for nogil Python). In 3.11+, the argval for LOAD_GLOBAL would need to be divided by two to access the correct name. The 'argval' field already stores the correct name. - Set '__builtins__' before constructing de-pickled functions. (Useful for nogil Python) - Fix test_recursion_during_pickling in Python 3.11+. Objects now have a default `__getstate__` method so `__getattr__` was never called, but `__getattribute__` would still be called. --- cloudpickle/cloudpickle.py | 13 ++++++++----- cloudpickle/cloudpickle_fast.py | 23 +++++++++++++++++------ tests/cloudpickle_test.py | 2 +- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 6fb4462d..65fb0073 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -321,11 +321,10 @@ def _extract_code_globals(co): """ out_names = _extract_code_globals_cache.get(co) if out_names is None: - names = co.co_names # We use a dict with None values instead of a set to get a # deterministic order (assuming Python 3.6+) and avoid introducing # non-deterministic pickle bytes as a results. - out_names = {names[oparg]: None for _, oparg in _walk_global_ops(co)} + out_names = {name: None for name in _walk_global_ops(co)} # Declaring a function inside another one using the "def ..." # syntax generates a constant code object corresponding to the one @@ -511,13 +510,12 @@ def _builtin_type(name): def _walk_global_ops(code): """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. + Yield referenced name for all global-referencing instructions in *code*. """ for instr in dis.get_instructions(code): op = instr.opcode if op in GLOBAL_OPS: - yield op, instr.arg + yield instr.argval def _extract_class_dict(cls): @@ -765,6 +763,11 @@ def _fill_function(*args): return func +def _make_function(code, globals, name, argdefs, closure): + globals["__builtins__"] = __builtins__ + return types.FunctionType(code, globals, name, argdefs, closure) + + def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 99522a7f..afb450cf 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -35,7 +35,7 @@ _is_parametrized_type_hint, PYPY, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, - _make_dict_keys, _make_dict_values, _make_dict_items, + _make_dict_keys, _make_dict_values, _make_dict_items, _make_function, ) @@ -248,7 +248,7 @@ def _code_reduce(obj): # of the specific type from types, for example: # >>> from types import CodeType # >>> help(CodeType) - if hasattr(obj, "co_columntable"): # pragma: no branch + if hasattr(obj, "co_exceptiontable"): # pragma: no branch # Python 3.11 and later: there are some new attributes # related to the enhanced exceptions. args = ( @@ -256,9 +256,8 @@ def _code_reduce(obj): obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - obj.co_cellvars, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, + obj.co_freevars, obj.co_cellvars, ) elif hasattr(obj, "co_linetable"): # pragma: no branch # Python 3.10 and later: obj.co_lnotab is deprecated and constructor @@ -271,6 +270,18 @@ def _code_reduce(obj): obj.co_firstlineno, obj.co_linetable, obj.co_freevars, obj.co_cellvars ) + elif hasattr(obj, "co_nmeta"): # pragma: no branch + # "nogil" Python: modified attributes from 3.9 + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_framesize, + obj.co_ndefaultargs, obj.co_nmeta, + obj.co_flags, obj.co_code, obj.co_consts, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_exc_handlers, + obj.co_jump_table, obj.co_freevars, obj.co_cellvars, + obj.co_free2reg, obj.co_cell2reg + ) elif hasattr(obj, "co_posonlyargcount"): # Backward compat for 3.9 and older args = ( @@ -564,7 +575,7 @@ def _dynamic_function_reduce(self, func): """Reduce a function that is not pickleable via attribute lookup.""" newargs = self._function_getnewargs(func) state = _function_getstate(func) - return (types.FunctionType, newargs, state, None, None, + return (_make_function, newargs, state, None, None, _function_setstate) def _function_reduce(self, obj): diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index c56e7e8c..9f672c5e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2245,7 +2245,7 @@ def inner_function(): def test_recursion_during_pickling(self): class A: - def __getattr__(self, name): + def __getattribute__(self, name): return getattr(self, name) a = A() From 3e0d5fc2f1a7295a853e3210455a4e4c6f800b7d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 20 May 2022 11:13:00 +0200 Subject: [PATCH 2/6] TST skip test_extended_arg on 3.11.0b1 --- tests/cloudpickle_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 9f672c5e..22904932 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -978,6 +978,10 @@ def g(y): res = loop.run_sync(functools.partial(g2, 5)) self.assertEqual(res, 7) + @pytest.mark.skipif( + (3, 11, 0, 'beta') <= sys.version_info < (3, 11, 0, 'beta', 2), + reason="https://github.com/python/cpython/issues/92932" + ) def test_extended_arg(self): # Functions with more than 65535 global vars prefix some global # variable references with the EXTENDED_ARG opcode. From 400dd9aa007ecaca76402f69e104c9824fee1062 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 20 May 2022 11:32:28 +0200 Subject: [PATCH 3/6] DOC add comment in _make_function --- cloudpickle/cloudpickle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 65fb0073..317be691 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -764,6 +764,7 @@ def _fill_function(*args): def _make_function(code, globals, name, argdefs, closure): + # Setting __builtins__ in globals is needed for nogil CPython. globals["__builtins__"] = __builtins__ return types.FunctionType(code, globals, name, argdefs, closure) From f0cf0f79361f002b95cfbaf86a0db13817cd4242 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 20 May 2022 11:43:57 +0200 Subject: [PATCH 4/6] Trigger CI From 5e91b4c0d7065de2b9e96e1468b2eb819c592871 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 20 May 2022 12:08:48 +0200 Subject: [PATCH 5/6] TST trying to get coverage not to fail on CI --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index afb450cf..a0fd44a1 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -270,7 +270,7 @@ def _code_reduce(obj): obj.co_firstlineno, obj.co_linetable, obj.co_freevars, obj.co_cellvars ) - elif hasattr(obj, "co_nmeta"): # pragma: no branch + elif hasattr(obj, "co_nmeta"): # pragma: no cover # "nogil" Python: modified attributes from 3.9 args = ( obj.co_argcount, obj.co_posonlyargcount, From bb3c31b31e5122909715e3128365e778c3c6ec61 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 20 May 2022 15:17:39 +0200 Subject: [PATCH 6/6] Document change in the changelog --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c0c2873c..de5cc262 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ - Support and CI configuration for Python 3.11. ([PR #467](https://github.com/cloudpipe/cloudpickle/pull/467)) +- Support for the experimental `nogil` variant of CPython + ([PR #470](https://github.com/cloudpipe/cloudpickle/pull/470)) + 2.0.0 =====