From 669f6be85b239711f7fb4d917153d9f0900b0d1c Mon Sep 17 00:00:00 2001 From: Stephen Macke Date: Wed, 2 Jun 2021 12:18:18 -0700 Subject: [PATCH 1/4] support OrderedDict --- cloudpickle/cloudpickle.py | 22 ++++++++++++++++------ cloudpickle/cloudpickle_fast.py | 23 ++++++++++++++++++++++- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 20e9a9550..3470ca680 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -55,6 +55,7 @@ import warnings from .compat import pickle +from collections import OrderedDict from typing import Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec @@ -852,13 +853,22 @@ def _get_bases(typ): return getattr(typ, bases_attr) -def _make_dict_keys(obj): - return dict.fromkeys(obj).keys() +def _make_dict_keys(obj, is_ordered=False): + if is_ordered: + return OrderedDict.fromkeys(obj).keys() + else: + return dict.fromkeys(obj).keys() -def _make_dict_values(obj): - return {i: _ for i, _ in enumerate(obj)}.values() +def _make_dict_values(obj, is_ordered=False): + if is_ordered: + return OrderedDict((i, _) for i, _ in enumerate(obj)).values() + else: + return {i: _ for i, _ in enumerate(obj)}.values() -def _make_dict_items(obj): - return obj.items() +def _make_dict_items(obj, is_ordered=False): + if is_ordered: + return OrderedDict(obj).items() + else: + return obj.items() diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 46a9540ec..91534ffea 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -23,7 +23,7 @@ import typing from enum import Enum -from collections import ChainMap +from collections import ChainMap, OrderedDict from .compat import pickle, Pickler from .cloudpickle import ( @@ -419,6 +419,24 @@ def _dict_items_reduce(obj): return _make_dict_items, (dict(obj), ) +def _odict_keys_reduce(obj): + # Safer not to ship the full dict as sending the rest might + # be unintended and could potentially cause leaking of + # sensitive information + return _make_dict_keys, (list(obj), True) + + +def _odict_values_reduce(obj): + # Safer not to ship the full dict as sending the rest might + # be unintended and could potentially cause leaking of + # sensitive information + return _make_dict_values, (list(obj), True) + + +def _odict_items_reduce(obj): + return _make_dict_items, (dict(obj), True) + + # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ # state setters are called at unpickling time, once the object is created and @@ -495,6 +513,9 @@ class CloudPickler(Pickler): _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce + _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce + _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce + _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) From cbb2828a99e9b546502c6aeac1c0ea9038bbf907 Mon Sep 17 00:00:00 2001 From: Stephen Macke Date: Wed, 2 Jun 2021 12:22:49 -0700 Subject: [PATCH 2/4] add tests for OrderedDict keys / values / items --- tests/cloudpickle_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 845f27962..d9874e536 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -226,6 +226,24 @@ def test_dict_items(self): self.assertEqual(results, items) assert isinstance(results, _collections_abc.dict_items) + def test_odict_keys(self): + keys = collections.OrderedDict([("a", 1), ("b", 2)]).keys() + results = pickle_depickle(keys) + self.assertEqual(results, keys) + assert type(keys) == type(results) + + def test_odict_values(self): + values = collections.OrderedDict([("a", 1), ("b", 2)]).values() + results = pickle_depickle(values) + self.assertEqual(sorted(results), sorted(values)) + assert type(values) == type(results) + + def test_odict_items(self): + items = collections.OrderedDict([("a", 1), ("b", 2)]).items() + results = pickle_depickle(items) + self.assertEqual(results, items) + assert type(items) == type(results) + def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), From 701e2e24010a64c505b383645e681831208ef609 Mon Sep 17 00:00:00 2001 From: Stephen Macke Date: Tue, 22 Jun 2021 10:38:41 -0700 Subject: [PATCH 3/4] update changelog --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c9b3d9384..f4d851468 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,10 @@ dev _is_parametrized_type_hint to limit false positives. ([PR #409](https://github.com/cloudpipe/cloudpickle/pull/409)) +- Support pickling / depickling of OrderedDict KeysView, ValuesView, and + ItemsView, following similar strategy for vanilla Python dictionaries. + ([PR #423](https://github.com/cloudpipe/cloudpickle/pull/423)) + 1.6.0 ===== From a1eb14f3187b4137b00545cc1da33493e2ed75b3 Mon Sep 17 00:00:00 2001 From: Stephen Macke Date: Wed, 30 Jun 2021 12:36:15 -0700 Subject: [PATCH 4/4] leave ordering of values() unchanged when pulling them out for tests --- tests/cloudpickle_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index d9874e536..798afecde 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -235,7 +235,7 @@ def test_odict_keys(self): def test_odict_values(self): values = collections.OrderedDict([("a", 1), ("b", 2)]).values() results = pickle_depickle(values) - self.assertEqual(sorted(results), sorted(values)) + self.assertEqual(list(results), list(values)) assert type(values) == type(results) def test_odict_items(self):