From ada4b55e4c2bf998da85ffdcd9183029925d9e54 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Wed, 8 Jul 2020 12:06:28 -0600 Subject: [PATCH 1/4] Track modules that should be treated as dynamic Create a set that will be used to declare modules whose resources should be treated as dynamic during pickling. The default behaviour is to reference module-backed resources, instead of pickling them, which is inconvenient in certain use cases. --- cloudpickle/cloudpickle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 8e683e7a6..6d7e85418 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -87,6 +87,9 @@ def g(): # communication speed over compatibility: DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL +# Names of modules whose resources should be treated as dynamic. +_CUSTOM_DYNAMIC_MODULES_BY_NAME = set() + # Track the provenance of reconstructed dynamic classes to make it possible to # recontruct instances from the matching singleton class definition when # appropriate and preserve the usual "isinstance" semantics of Python objects. From 0858ab2a8adcc1126052c167308ae268956f9780 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Wed, 8 Jul 2020 12:19:57 -0600 Subject: [PATCH 2/4] Add functions for registering dynamic modules register_dynamic_module takes a module object or a module name and adds it to the _CUSTOM_DYNAMIC_MODULES_BY_NAME set. Registered modules can be remove with unregister_dynamic_module. --- cloudpickle/cloudpickle.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 6d7e85418..079b0d931 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -44,6 +44,7 @@ import builtins import dis +import inspect import opcode import platform import sys @@ -126,6 +127,16 @@ def _lookup_class_or_track(class_tracker_id, class_def): return class_def +def register_dynamic_module(module): + module_name = module.__name__ if inspect.ismodule(module) else module + _CUSTOM_DYNAMIC_MODULES_BY_NAME.add(module_name) + + +def unregister_dynamic_module(module): + module_name = module.__name__ if inspect.ismodule(module) else module + _CUSTOM_DYNAMIC_MODULES_BY_NAME.remove(module_name) + + def _whichmodule(obj, name): """Find the module an object belongs to. From 79b82df699862f38be9539b90a6c3c385b245027 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Wed, 8 Jul 2020 12:34:43 -0600 Subject: [PATCH 3/4] Add function to determine if module is dynamic _is_dynamic_module returns True if the specified module is in the set of modules that should be considered dynamic for pickling purposes. If submodules is True, then the module's parent modules will be searched to see if they were registered as dynamic. --- cloudpickle/cloudpickle.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 079b0d931..fcad8be4b 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -137,6 +137,21 @@ def unregister_dynamic_module(module): _CUSTOM_DYNAMIC_MODULES_BY_NAME.remove(module_name) +def _is_dynamic_module(module, submodules=True): + module_name = module.__name__ if inspect.ismodule(module) else module + if module_name in _CUSTOM_DYNAMIC_MODULES_BY_NAME: + return True + if submodules: + while True: + parent_name = module_name.rsplit(".", 1)[0] + if parent_name == module_name: + break + if parent_name in _CUSTOM_DYNAMIC_MODULES_BY_NAME: + return True + module_name = parent_name + return False + + def _whichmodule(obj, name): """Find the module an object belongs to. From f789f2f216f4cbf3c20e220f80f94fed82a52186 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Wed, 8 Jul 2020 12:36:41 -0600 Subject: [PATCH 4/4] Check for dynamic module Add a condition while looking up modules to see if the module is registered as dynamic, returning None if so. This means resources belonging to modules registered as dynamic will be treated as if they had no module, or belonged to __main__. --- cloudpickle/cloudpickle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index fcad8be4b..6043fd8ab 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -234,6 +234,9 @@ def _lookup_module_and_qualname(obj, name=None): if module_name == "__main__": return None + if _is_dynamic_module(module_name): + return None + # Note: if module_name is in sys.modules, the corresponding module is # assumed importable at unpickling time. See #357 module = sys.modules.get(module_name, None)