From 15e545abac80dfaad443621fb35a2478eeae9282 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 23 Feb 2022 16:57:03 -0800 Subject: [PATCH 01/28] Integrate orjson into MontyDecoder --- monty/json.py | 89 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/monty/json.py b/monty/json.py index e566d5bc..88854c6c 100644 --- a/monty/json.py +++ b/monty/json.py @@ -12,6 +12,7 @@ from importlib import import_module from inspect import getfullargspec from uuid import UUID +import orjson try: import numpy as np @@ -171,7 +172,11 @@ def from_dict(cls, d): :param d: Dict representation. :return: MSONable class. """ - decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items() if not k.startswith("@")} + decoded = { + k: MontyDecoder().process_decoded(v) + for k, v in d.items() + if not k.startswith("@") + } return cls(**decoded) def to_json(self) -> str: @@ -193,16 +198,25 @@ def flatten(obj, seperator="."): flat_dict = {} for key, value in obj.items(): if isinstance(value, dict): - flat_dict.update({seperator.join([key, _key]): _value for _key, _value in flatten(value).items()}) + flat_dict.update( + { + seperator.join([key, _key]): _value + for _key, _value in flatten(value).items() + } + ) elif isinstance(value, list): - list_dict = {f"{key}{seperator}{num}": item for num, item in enumerate(value)} + list_dict = { + f"{key}{seperator}{num}": item for num, item in enumerate(value) + } flat_dict.update(flatten(list_dict)) else: flat_dict[key] = value return flat_dict - ordered_keys = sorted(flatten(jsanitize(self.as_dict())).items(), key=lambda x: x[0]) + ordered_keys = sorted( + flatten(jsanitize(self.as_dict())).items(), key=lambda x: x[0] + ) ordered_keys = [item for item in ordered_keys if "@" not in item[0]] return sha1(json.dumps(OrderedDict(ordered_keys)).encode("utf-8")) @@ -226,7 +240,9 @@ def validate_monty(cls, v): new_obj = cls(**v) return new_obj - raise ValueError(f"Must provide {cls.__name__}, the as_dict form, or the proper") + raise ValueError( + f"Must provide {cls.__name__}, the as_dict form, or the proper" + ) @classmethod def __modify_schema__(cls, field_schema): @@ -335,7 +351,7 @@ def default(self, o) -> dict: # pylint: disable=E0202 return json.JSONEncoder.default(self, o) -class MontyDecoder(json.JSONDecoder): +class MontyDecoder: """ A Json Decoder which supports the MSONable API. By default, the decoder attempts to find a module and name associated with a dict. If @@ -396,9 +412,13 @@ def process_decoded(self, d): if modname and modname not in ["bson.objectid", "numpy", "pandas"]: if modname == "datetime" and classname == "datetime": try: - dt = datetime.datetime.strptime(d["string"], "%Y-%m-%d %H:%M:%S.%f") + dt = datetime.datetime.strptime( + d["string"], "%Y-%m-%d %H:%M:%S.%f" + ) except ValueError: - dt = datetime.datetime.strptime(d["string"], "%Y-%m-%d %H:%M:%S") + dt = datetime.datetime.strptime( + d["string"], "%Y-%m-%d %H:%M:%S" + ) return dt if modname == "uuid" and classname == "UUID": @@ -410,12 +430,17 @@ def process_decoded(self, d): data = {k: v for k, v in d.items() if not k.startswith("@")} if hasattr(cls_, "from_dict"): return cls_.from_dict(data) - if pydantic is not None and issubclass(cls_, pydantic.BaseModel): + if pydantic is not None and issubclass( + cls_, pydantic.BaseModel + ): return cls_(**data) elif np is not None and modname == "numpy" and classname == "array": if d["dtype"].startswith("complex"): return np.array( - [np.array(r) + np.array(i) * 1j for r, i in zip(*d["data"])], + [ + np.array(r) + np.array(i) * 1j + for r, i in zip(*d["data"]) + ], dtype=d["dtype"], ) return np.array(d["data"], dtype=d["dtype"]) @@ -426,10 +451,16 @@ def process_decoded(self, d): if classname == "Series": decoded_data = MontyDecoder().decode(d["data"]) return pd.Series(decoded_data) - elif (bson is not None) and modname == "bson.objectid" and classname == "ObjectId": + elif ( + (bson is not None) + and modname == "bson.objectid" + and classname == "ObjectId" + ): return bson.objectid.ObjectId(d["oid"]) - return {self.process_decoded(k): self.process_decoded(v) for k, v in d.items()} + return { + self.process_decoded(k): self.process_decoded(v) for k, v in d.items() + } if isinstance(d, list): return [self.process_decoded(x) for x in d] @@ -443,7 +474,7 @@ def decode(self, s): :param s: string :return: Object. """ - d = json.JSONDecoder.decode(self, s) + d = orjson.loads(s) return self.process_decoded(d) @@ -481,20 +512,29 @@ def jsanitize(obj, strict=False, allow_bson=False, enum_values=False): return obj.value if allow_bson and ( - isinstance(obj, (datetime.datetime, bytes)) or (bson is not None and isinstance(obj, bson.objectid.ObjectId)) + isinstance(obj, (datetime.datetime, bytes)) + or (bson is not None and isinstance(obj, bson.objectid.ObjectId)) ): return obj if isinstance(obj, (list, tuple)): - return [jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) for i in obj] + return [ + jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) + for i in obj + ] if np is not None and isinstance(obj, np.ndarray): - return [jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) for i in obj.tolist()] + return [ + jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) + for i in obj.tolist() + ] if np is not None and isinstance(obj, np.generic): return obj.item() if pd is not None and isinstance(obj, pd.DataFrame) or isinstance(obj, pd.Series): return obj.to_dict() if isinstance(obj, dict): return { - k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson, enum_values=enum_values) + k.__str__(): jsanitize( + v, strict=strict, allow_bson=allow_bson, enum_values=enum_values + ) for k, v in obj.items() } if isinstance(obj, (int, float)): @@ -515,9 +555,16 @@ def jsanitize(obj, strict=False, allow_bson=False, enum_values=False): return obj.__str__() if pydantic is not None and isinstance(obj, pydantic.BaseModel): - return jsanitize(MontyEncoder().default(obj), strict=strict, allow_bson=allow_bson, enum_values=enum_values) + return jsanitize( + MontyEncoder().default(obj), + strict=strict, + allow_bson=allow_bson, + enum_values=enum_values, + ) - return jsanitize(obj.as_dict(), strict=strict, allow_bson=allow_bson, enum_values=enum_values) + return jsanitize( + obj.as_dict(), strict=strict, allow_bson=allow_bson, enum_values=enum_values + ) def _serialize_callable(o): @@ -535,7 +582,9 @@ def _serialize_callable(o): try: bound = MontyEncoder().default(bound) except TypeError: - raise TypeError("Only bound methods of classes or MSONable instances are supported.") + raise TypeError( + "Only bound methods of classes or MSONable instances are supported." + ) return { "@module": o.__module__, From 2490305c03c5a109265e00d4fd02360c6eeba710 Mon Sep 17 00:00:00 2001 From: munrojm Date: Thu, 24 Feb 2022 14:12:49 -0800 Subject: [PATCH 02/28] Subclass JSONDecoder --- monty/json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/json.py b/monty/json.py index 88854c6c..1160059c 100644 --- a/monty/json.py +++ b/monty/json.py @@ -351,7 +351,7 @@ def default(self, o) -> dict: # pylint: disable=E0202 return json.JSONEncoder.default(self, o) -class MontyDecoder: +class MontyDecoder(json.JSONDecoder): """ A Json Decoder which supports the MSONable API. By default, the decoder attempts to find a module and name associated with a dict. If From f5fe54802e9c4e45760bff1319910304155b2256 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 7 Mar 2022 16:00:41 -0800 Subject: [PATCH 03/28] Change serialization funcs --- monty/serialization.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/monty/serialization.py b/monty/serialization.py index 331c23b6..04a3b003 100644 --- a/monty/serialization.py +++ b/monty/serialization.py @@ -2,7 +2,7 @@ This module implements serialization support for common formats such as json and yaml. """ -import json +import orjson import os try: @@ -33,11 +33,11 @@ def loadfn(fn, *args, fmt=None, **kwargs): Args: fn (str/Path): filename or pathlib.Path. - *args: Any of the args supported by json/yaml.load. + *args: Any of the args supported by msgpack/yaml.load. fmt (string): If specified, the fmt specified would be used instead of autodetection from filename. Supported formats right now are "json", "yaml" or "mpk". - **kwargs: Any of the kwargs supported by json/yaml.load. + **kwargs: Any of the kwargs supported by msgpack/yaml.load. Returns: (object) Result of json/yaml/msgpack.load. @@ -69,8 +69,10 @@ def loadfn(fn, *args, fmt=None, **kwargs): if fmt == "json": if "cls" not in kwargs: kwargs["cls"] = MontyDecoder - return json.load(fp, *args, **kwargs) + file_content = fp.read() + bo = orjson.loads(file_content) + return bo raise TypeError(f"Invalid format: {fmt}") @@ -88,8 +90,8 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): Args: obj (object): Object to dump. fn (str/Path): filename or pathlib.Path. - *args: Any of the args supported by json/yaml.dump. - **kwargs: Any of the kwargs supported by json/yaml.dump. + *args: Any of the args supported by msgpack/yaml.dump. + **kwargs: Any of the kwargs supported by msgpack/yaml.dump. Returns: (object) Result of json.load. @@ -120,6 +122,7 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): elif fmt == "json": if "cls" not in kwargs: kwargs["cls"] = MontyEncoder - fp.write(json.dumps(obj, *args, **kwargs)) + so = orjson.dumps(obj) + fp.write(so.decode("utf-8")) else: raise TypeError(f"Invalid format: {fmt}") From e790ac13a4ed88fbc7e4b904e72f2c089bb67731 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 7 Mar 2022 16:05:18 -0800 Subject: [PATCH 04/28] Revert serialization change --- monty/serialization.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/monty/serialization.py b/monty/serialization.py index 04a3b003..5acbc43c 100644 --- a/monty/serialization.py +++ b/monty/serialization.py @@ -2,7 +2,7 @@ This module implements serialization support for common formats such as json and yaml. """ -import orjson +import json import os try: @@ -33,11 +33,11 @@ def loadfn(fn, *args, fmt=None, **kwargs): Args: fn (str/Path): filename or pathlib.Path. - *args: Any of the args supported by msgpack/yaml.load. + *args: Any of the args supported by json/yaml.load. fmt (string): If specified, the fmt specified would be used instead of autodetection from filename. Supported formats right now are "json", "yaml" or "mpk". - **kwargs: Any of the kwargs supported by msgpack/yaml.load. + **kwargs: Any of the kwargs supported by json/yaml.load. Returns: (object) Result of json/yaml/msgpack.load. @@ -54,7 +54,9 @@ def loadfn(fn, *args, fmt=None, **kwargs): if fmt == "mpk": if msgpack is None: - raise RuntimeError("Loading of message pack files is not possible as msgpack-python is not installed.") + raise RuntimeError( + "Loading of message pack files is not possible as msgpack-python is not installed." + ) if "object_hook" not in kwargs: kwargs["object_hook"] = object_hook with zopen(fn, "rb") as fp: @@ -69,10 +71,8 @@ def loadfn(fn, *args, fmt=None, **kwargs): if fmt == "json": if "cls" not in kwargs: kwargs["cls"] = MontyDecoder + return json.load(fp, *args, **kwargs) - file_content = fp.read() - bo = orjson.loads(file_content) - return bo raise TypeError(f"Invalid format: {fmt}") @@ -90,8 +90,8 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): Args: obj (object): Object to dump. fn (str/Path): filename or pathlib.Path. - *args: Any of the args supported by msgpack/yaml.dump. - **kwargs: Any of the kwargs supported by msgpack/yaml.dump. + *args: Any of the args supported by json/yaml.dump. + **kwargs: Any of the kwargs supported by json/yaml.dump. Returns: (object) Result of json.load. @@ -107,7 +107,9 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): if fmt == "mpk": if msgpack is None: - raise RuntimeError("Loading of message pack files is not possible as msgpack-python is not installed.") + raise RuntimeError( + "Loading of message pack files is not possible as msgpack-python is not installed." + ) if "default" not in kwargs: kwargs["default"] = default with zopen(fn, "wb") as fp: @@ -122,7 +124,6 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): elif fmt == "json": if "cls" not in kwargs: kwargs["cls"] = MontyEncoder - so = orjson.dumps(obj) - fp.write(so.decode("utf-8")) + fp.write(json.dumps(obj, *args, **kwargs)) else: raise TypeError(f"Invalid format: {fmt}") From 5b45b6e82d5c095ba493e0414f88e34062ac7e68 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 7 Mar 2022 16:41:34 -0800 Subject: [PATCH 05/28] Initial commit of new encode func --- monty/json.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/monty/json.py b/monty/json.py index 1160059c..b10e30ed 100644 --- a/monty/json.py +++ b/monty/json.py @@ -271,7 +271,28 @@ class MontyEncoder(json.JSONEncoder): json.dumps(object, cls=MontyEncoder) """ - def default(self, o) -> dict: # pylint: disable=E0202 + def encode(self, o): + if not isinstance(o, (list, tuple)): + d = self.process(o) + else: + d = self.handle_iters(o) + + e = orjson.dumps(d).decode("utf-8") + return e + + def handle_iters(self, o): + d_list = [] + for item in o: + try: + d = self.process(item) + d_list.append(d) + except TypeError: + d = self.handle_iters(item) + d_list.append(d) + + return d_list + + def process(self, o) -> dict: # pylint: disable=E0202 """ Overriding default method for JSON encoding. This method does two things: (a) If an object has a to_dict property, return the to_dict From ae76728559fc01567886c3a60320152100513066 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 11:59:27 -0800 Subject: [PATCH 06/28] Add new encode func --- monty/json.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/monty/json.py b/monty/json.py index b10e30ed..68e08c63 100644 --- a/monty/json.py +++ b/monty/json.py @@ -272,22 +272,27 @@ class MontyEncoder(json.JSONEncoder): """ def encode(self, o): - if not isinstance(o, (list, tuple)): - d = self.process(o) - else: + if isinstance(o, (list, tuple, dict)): d = self.handle_iters(o) + else: + d = self.process(o) e = orjson.dumps(d).decode("utf-8") return e def handle_iters(self, o): - d_list = [] + d_list = {} if isinstance(o, dict) else [] + for item in o: + val = o[item] if isinstance(d_list, dict) else item try: - d = self.process(item) - d_list.append(d) + d = self.process(val) except TypeError: - d = self.handle_iters(item) + d = self.handle_iters(val) + + if isinstance(d_list, dict): + d_list[item] = d + else: d_list.append(d) return d_list @@ -367,9 +372,13 @@ def process(self, o) -> dict: # pylint: disable=E0202 d["@version"] = str(module_version) except (AttributeError, ImportError): d["@version"] = None - return d + + d_p = self.handle_iters(d) + + return d_p except AttributeError: - return json.JSONEncoder.default(self, o) + return o + # return json.JSONEncoder.default(self, o) class MontyDecoder(json.JSONDecoder): @@ -577,7 +586,7 @@ def jsanitize(obj, strict=False, allow_bson=False, enum_values=False): if pydantic is not None and isinstance(obj, pydantic.BaseModel): return jsanitize( - MontyEncoder().default(obj), + MontyEncoder().process(obj), strict=strict, allow_bson=allow_bson, enum_values=enum_values, @@ -601,7 +610,7 @@ def _serialize_callable(o): # bound to is itself serializable if bound is not None: try: - bound = MontyEncoder().default(bound) + bound = MontyEncoder().process(bound) except TypeError: raise TypeError( "Only bound methods of classes or MSONable instances are supported." From 19a3ce5a3d39f850275eeafb37183a35c775cfa5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 12:42:56 -0800 Subject: [PATCH 07/28] Fix serializing bound methods --- monty/json.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/monty/json.py b/monty/json.py index 68e08c63..c0362425 100644 --- a/monty/json.py +++ b/monty/json.py @@ -272,6 +272,15 @@ class MontyEncoder(json.JSONEncoder): """ def encode(self, o): + """ + Overrides encode method on default JSONEncoder. + + Args: + o: Python object. + + Return: + JSON string representation. + """ if isinstance(o, (list, tuple, dict)): d = self.handle_iters(o) else: @@ -281,6 +290,15 @@ def encode(self, o): return e def handle_iters(self, o): + """ + Recursive function for handling encoding of iterables + + Args: + o: Iterable Python object. + + Return: + list of Python dict representations. + """ d_list = {} if isinstance(o, dict) else [] for item in o: @@ -299,7 +317,7 @@ def handle_iters(self, o): def process(self, o) -> dict: # pylint: disable=E0202 """ - Overriding default method for JSON encoding. This method does two + Processing for sanitization before encoding. This method does two things: (a) If an object has a to_dict property, return the to_dict output. (b) If the @module and @class keys are not in the to_dict, add them to the output automatically. If the object has no to_dict @@ -378,7 +396,6 @@ def process(self, o) -> dict: # pylint: disable=E0202 return d_p except AttributeError: return o - # return json.JSONEncoder.default(self, o) class MontyDecoder(json.JSONDecoder): @@ -609,12 +626,14 @@ def _serialize_callable(o): # we are only able to serialize bound methods if the object the method is # bound to is itself serializable if bound is not None: - try: - bound = MontyEncoder().process(bound) - except TypeError: + + new_bound = MontyEncoder().process(bound) + if new_bound == bound: raise TypeError( "Only bound methods of classes or MSONable instances are supported." ) + else: + bound = new_bound return { "@module": o.__module__, From f428193081643df433aeef24f1e0b57b489ed527 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 12:54:41 -0800 Subject: [PATCH 08/28] Linting --- monty/json.py | 77 ++++++++++++--------------------------------------- 1 file changed, 17 insertions(+), 60 deletions(-) diff --git a/monty/json.py b/monty/json.py index c0362425..cf68a80c 100644 --- a/monty/json.py +++ b/monty/json.py @@ -172,11 +172,7 @@ def from_dict(cls, d): :param d: Dict representation. :return: MSONable class. """ - decoded = { - k: MontyDecoder().process_decoded(v) - for k, v in d.items() - if not k.startswith("@") - } + decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items() if not k.startswith("@")} return cls(**decoded) def to_json(self) -> str: @@ -198,25 +194,16 @@ def flatten(obj, seperator="."): flat_dict = {} for key, value in obj.items(): if isinstance(value, dict): - flat_dict.update( - { - seperator.join([key, _key]): _value - for _key, _value in flatten(value).items() - } - ) + flat_dict.update({seperator.join([key, _key]): _value for _key, _value in flatten(value).items()}) elif isinstance(value, list): - list_dict = { - f"{key}{seperator}{num}": item for num, item in enumerate(value) - } + list_dict = {f"{key}{seperator}{num}": item for num, item in enumerate(value)} flat_dict.update(flatten(list_dict)) else: flat_dict[key] = value return flat_dict - ordered_keys = sorted( - flatten(jsanitize(self.as_dict())).items(), key=lambda x: x[0] - ) + ordered_keys = sorted(flatten(jsanitize(self.as_dict())).items(), key=lambda x: x[0]) ordered_keys = [item for item in ordered_keys if "@" not in item[0]] return sha1(json.dumps(OrderedDict(ordered_keys)).encode("utf-8")) @@ -240,9 +227,7 @@ def validate_monty(cls, v): new_obj = cls(**v) return new_obj - raise ValueError( - f"Must provide {cls.__name__}, the as_dict form, or the proper" - ) + raise ValueError(f"Must provide {cls.__name__}, the as_dict form, or the proper") @classmethod def __modify_schema__(cls, field_schema): @@ -459,13 +444,9 @@ def process_decoded(self, d): if modname and modname not in ["bson.objectid", "numpy", "pandas"]: if modname == "datetime" and classname == "datetime": try: - dt = datetime.datetime.strptime( - d["string"], "%Y-%m-%d %H:%M:%S.%f" - ) + dt = datetime.datetime.strptime(d["string"], "%Y-%m-%d %H:%M:%S.%f") except ValueError: - dt = datetime.datetime.strptime( - d["string"], "%Y-%m-%d %H:%M:%S" - ) + dt = datetime.datetime.strptime(d["string"], "%Y-%m-%d %H:%M:%S") return dt if modname == "uuid" and classname == "UUID": @@ -477,17 +458,12 @@ def process_decoded(self, d): data = {k: v for k, v in d.items() if not k.startswith("@")} if hasattr(cls_, "from_dict"): return cls_.from_dict(data) - if pydantic is not None and issubclass( - cls_, pydantic.BaseModel - ): + if pydantic is not None and issubclass(cls_, pydantic.BaseModel): return cls_(**data) elif np is not None and modname == "numpy" and classname == "array": if d["dtype"].startswith("complex"): return np.array( - [ - np.array(r) + np.array(i) * 1j - for r, i in zip(*d["data"]) - ], + [np.array(r) + np.array(i) * 1j for r, i in zip(*d["data"])], dtype=d["dtype"], ) return np.array(d["data"], dtype=d["dtype"]) @@ -498,16 +474,10 @@ def process_decoded(self, d): if classname == "Series": decoded_data = MontyDecoder().decode(d["data"]) return pd.Series(decoded_data) - elif ( - (bson is not None) - and modname == "bson.objectid" - and classname == "ObjectId" - ): + elif (bson is not None) and modname == "bson.objectid" and classname == "ObjectId": return bson.objectid.ObjectId(d["oid"]) - return { - self.process_decoded(k): self.process_decoded(v) for k, v in d.items() - } + return {self.process_decoded(k): self.process_decoded(v) for k, v in d.items()} if isinstance(d, list): return [self.process_decoded(x) for x in d] @@ -559,29 +529,20 @@ def jsanitize(obj, strict=False, allow_bson=False, enum_values=False): return obj.value if allow_bson and ( - isinstance(obj, (datetime.datetime, bytes)) - or (bson is not None and isinstance(obj, bson.objectid.ObjectId)) + isinstance(obj, (datetime.datetime, bytes)) or (bson is not None and isinstance(obj, bson.objectid.ObjectId)) ): return obj if isinstance(obj, (list, tuple)): - return [ - jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) - for i in obj - ] + return [jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) for i in obj] if np is not None and isinstance(obj, np.ndarray): - return [ - jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) - for i in obj.tolist() - ] + return [jsanitize(i, strict=strict, allow_bson=allow_bson, enum_values=enum_values) for i in obj.tolist()] if np is not None and isinstance(obj, np.generic): return obj.item() if pd is not None and isinstance(obj, pd.DataFrame) or isinstance(obj, pd.Series): return obj.to_dict() if isinstance(obj, dict): return { - k.__str__(): jsanitize( - v, strict=strict, allow_bson=allow_bson, enum_values=enum_values - ) + k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson, enum_values=enum_values) for k, v in obj.items() } if isinstance(obj, (int, float)): @@ -609,9 +570,7 @@ def jsanitize(obj, strict=False, allow_bson=False, enum_values=False): enum_values=enum_values, ) - return jsanitize( - obj.as_dict(), strict=strict, allow_bson=allow_bson, enum_values=enum_values - ) + return jsanitize(obj.as_dict(), strict=strict, allow_bson=allow_bson, enum_values=enum_values) def _serialize_callable(o): @@ -629,9 +588,7 @@ def _serialize_callable(o): new_bound = MontyEncoder().process(bound) if new_bound == bound: - raise TypeError( - "Only bound methods of classes or MSONable instances are supported." - ) + raise TypeError("Only bound methods of classes or MSONable instances are supported.") else: bound = new_bound From 58627ed83700798adf75aa17c54126a322ef1d8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 20:54:56 +0000 Subject: [PATCH 09/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monty/dev.py | 6 ++++-- monty/json.py | 1 + monty/serialization.py | 8 ++------ setup.py | 5 +++-- tasks.py | 8 ++++---- tests/test_bisect.py | 2 +- tests/test_collections.py | 5 +++-- tests/test_design_patterns.py | 4 ++-- tests/test_dev.py | 5 +++-- tests/test_fractions.py | 2 +- tests/test_functools.py | 16 +++++----------- tests/test_io.py | 11 +++-------- tests/test_json.py | 14 ++++++++------ tests/test_logging.py | 2 +- tests/test_multiprocessing.py | 2 +- tests/test_operator.py | 1 + tests/test_os.py | 5 ++--- tests/test_pprint.py | 4 ++-- tests/test_re.py | 3 ++- tests/test_serialization.py | 6 +++--- tests/test_shutil.py | 15 ++++----------- tests/test_string.py | 2 +- tests/test_tempfile.py | 4 ++-- tests/test_termcolor.py | 11 +++-------- 24 files changed, 62 insertions(+), 80 deletions(-) diff --git a/monty/dev.py b/monty/dev.py index 4f31f4b5..fbb94faf 100644 --- a/monty/dev.py +++ b/monty/dev.py @@ -139,7 +139,8 @@ def get_ncpus(): # jython try: - from java.lang import Runtime # pylint: disable=import-outside-toplevel + from java.lang import \ + Runtime # pylint: disable=import-outside-toplevel runtime = Runtime.getRuntime() res = runtime.availableProcessors() @@ -213,7 +214,8 @@ def install_excepthook(hook_type="color", **kwargs): 0 if hook is installed successfully. """ try: - from IPython.core import ultratb # pylint: disable=import-outside-toplevel + from IPython.core import \ + ultratb # pylint: disable=import-outside-toplevel except ImportError: warnings.warn("Cannot install excepthook, IPyhon.core.ultratb not available") return 1 diff --git a/monty/json.py b/monty/json.py index cf68a80c..07807360 100644 --- a/monty/json.py +++ b/monty/json.py @@ -12,6 +12,7 @@ from importlib import import_module from inspect import getfullargspec from uuid import UUID + import orjson try: diff --git a/monty/serialization.py b/monty/serialization.py index 5acbc43c..331c23b6 100644 --- a/monty/serialization.py +++ b/monty/serialization.py @@ -54,9 +54,7 @@ def loadfn(fn, *args, fmt=None, **kwargs): if fmt == "mpk": if msgpack is None: - raise RuntimeError( - "Loading of message pack files is not possible as msgpack-python is not installed." - ) + raise RuntimeError("Loading of message pack files is not possible as msgpack-python is not installed.") if "object_hook" not in kwargs: kwargs["object_hook"] = object_hook with zopen(fn, "rb") as fp: @@ -107,9 +105,7 @@ def dumpfn(obj, fn, *args, fmt=None, **kwargs): if fmt == "mpk": if msgpack is None: - raise RuntimeError( - "Loading of message pack files is not possible as msgpack-python is not installed." - ) + raise RuntimeError("Loading of message pack files is not possible as msgpack-python is not installed.") if "default" not in kwargs: kwargs["default"] = default with zopen(fn, "wb") as fp: diff --git a/setup.py b/setup.py index d4bb007f..f3dfaa86 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ -import os -from setuptools import setup, find_packages import io +import os + +from setuptools import find_packages, setup current_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/tasks.py b/tasks.py index dfdf1c1a..9d06ef24 100755 --- a/tasks.py +++ b/tasks.py @@ -5,17 +5,17 @@ """ +import datetime import glob -import requests import json import os -import datetime import re +import requests from invoke import task -from monty.os import cd -from monty import __version__ as ver +from monty import __version__ as ver +from monty.os import cd __author__ = "Shyue Ping Ong" __copyright__ = "Copyright 2012, The Materials Project" diff --git a/tests/test_bisect.py b/tests/test_bisect.py index 5e7107eb..3c0425fe 100644 --- a/tests/test_bisect.py +++ b/tests/test_bisect.py @@ -1,6 +1,6 @@ import unittest -from monty.bisect import index, find_lt, find_le, find_gt, find_ge +from monty.bisect import find_ge, find_gt, find_le, find_lt, index class FuncTestCase(unittest.TestCase): diff --git a/tests/test_collections.py b/tests/test_collections.py index bca27739..6469071d 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,7 +1,8 @@ -import unittest import os +import unittest -from monty.collections import frozendict, Namespace, AttrDict, FrozenAttrDict, tree +from monty.collections import (AttrDict, FrozenAttrDict, Namespace, frozendict, + tree) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_design_patterns.py b/tests/test_design_patterns.py index 76d94161..6a74ae65 100644 --- a/tests/test_design_patterns.py +++ b/tests/test_design_patterns.py @@ -1,7 +1,7 @@ -import unittest import pickle +import unittest -from monty.design_patterns import singleton, cached_class +from monty.design_patterns import cached_class, singleton class SingletonTest(unittest.TestCase): diff --git a/tests/test_dev.py b/tests/test_dev.py index 00f0ccc5..bbe561ca 100644 --- a/tests/test_dev.py +++ b/tests/test_dev.py @@ -1,7 +1,8 @@ +import multiprocessing import unittest import warnings -import multiprocessing -from monty.dev import deprecated, requires, get_ncpus, install_excepthook + +from monty.dev import deprecated, get_ncpus, install_excepthook, requires class A: diff --git a/tests/test_fractions.py b/tests/test_fractions.py index 21692d0c..2f03a75c 100644 --- a/tests/test_fractions.py +++ b/tests/test_fractions.py @@ -1,6 +1,6 @@ import unittest -from monty.fractions import gcd, lcm, gcd_float +from monty.fractions import gcd, gcd_float, lcm class FuncTestCase(unittest.TestCase): diff --git a/tests/test_functools.py b/tests/test_functools.py index f803f54d..19d47278 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -1,16 +1,10 @@ -import unittest -import sys import platform +import sys import time -from monty.functools import ( - lru_cache, - lazy_property, - return_if_raise, - return_none_if_raise, - timeout, - TimeoutError, - prof_main, -) +import unittest + +from monty.functools import (TimeoutError, lazy_property, lru_cache, prof_main, + return_if_raise, return_none_if_raise, timeout) class TestLRUCache(unittest.TestCase): diff --git a/tests/test_io.py b/tests/test_io.py index e11be429..7494772c 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,18 +1,13 @@ -import unittest import os +import unittest try: from pathlib import Path except ImportError: Path = None # type: ignore -from monty.io import ( - reverse_readline, - zopen, - FileLock, - FileLockException, - reverse_readfile, -) +from monty.io import (FileLock, FileLockException, reverse_readfile, + reverse_readline, zopen) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_json.py b/tests/test_json.py index bff3ee49..a920c5e5 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1,17 +1,19 @@ __version__ = "0.1" +import datetime +import json import os import unittest +from enum import Enum + import numpy as np -import json -import datetime import pandas as pd from bson.objectid import ObjectId -from enum import Enum + +from monty.json import (MontyDecoder, MontyEncoder, MSONable, _load_redirect, + jsanitize) from . import __version__ as tests_version -from monty.json import MSONable, MontyEncoder, MontyDecoder, jsanitize -from monty.json import _load_redirect test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_files") @@ -288,7 +290,7 @@ def test_datetime(self): self.assertEqual(type(d["dt"]), datetime.datetime) def test_uuid(self): - from uuid import uuid4, UUID + from uuid import UUID, uuid4 uuid = uuid4() jsonstr = json.dumps(uuid, cls=MontyEncoder) diff --git a/tests/test_logging.py b/tests/test_logging.py index 58d6eea4..c32b9a45 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,7 +1,7 @@ +import logging import unittest from io import StringIO -import logging from monty.logging import logged diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index 00baaccf..f4c76a16 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -1,7 +1,7 @@ import unittest +from math import sqrt from monty.multiprocessing import imap_tqdm -from math import sqrt class FuncCase(unittest.TestCase): diff --git a/tests/test_operator.py b/tests/test_operator.py index 6c63eb67..a186c3d2 100644 --- a/tests/test_operator.py +++ b/tests/test_operator.py @@ -1,4 +1,5 @@ import unittest + from monty.operator import operator_from_str diff --git a/tests/test_os.py b/tests/test_os.py index b8fa4726..b680195d 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -1,15 +1,14 @@ -import unittest import os import platform +import unittest -from monty.os.path import which, zpath, find_exts from monty.os import cd, makedirs_p +from monty.os.path import find_exts, which, zpath test_dir = os.path.join(os.path.dirname(__file__), "test_files") class PathTest(unittest.TestCase): - def test_zpath(self): fullzpath = zpath(os.path.join(test_dir, "myfile_gz")) self.assertEqual(os.path.join(test_dir, "myfile_gz.gz"), fullzpath) diff --git a/tests/test_pprint.py b/tests/test_pprint.py index 72b80549..4f2afff7 100644 --- a/tests/test_pprint.py +++ b/tests/test_pprint.py @@ -1,7 +1,7 @@ -from monty.pprint import pprint_table, draw_tree - import unittest +from monty.pprint import draw_tree, pprint_table + class PprintTableTest(unittest.TestCase): def test_print(self): diff --git a/tests/test_re.py b/tests/test_re.py index 5d33a5da..de4fae07 100644 --- a/tests/test_re.py +++ b/tests/test_re.py @@ -1,5 +1,6 @@ -import unittest import os +import unittest + from monty.re import regrep test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 1e6f5245..05b4532c 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,7 +1,7 @@ -import unittest -import os -import json import glob +import json +import os +import unittest try: import msgpack diff --git a/tests/test_shutil.py b/tests/test_shutil.py index c7f5cf81..2a79e94e 100644 --- a/tests/test_shutil.py +++ b/tests/test_shutil.py @@ -1,19 +1,12 @@ -import unittest import os -import shutil import platform +import shutil import tempfile +import unittest from gzip import GzipFile -from monty.shutil import ( - copy_r, - compress_file, - decompress_file, - compress_dir, - decompress_dir, - gzip_dir, - remove, -) +from monty.shutil import (compress_dir, compress_file, copy_r, decompress_dir, + decompress_file, gzip_dir, remove) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_string.py b/tests/test_string.py index f8861741..4f6a755a 100644 --- a/tests/test_string.py +++ b/tests/test_string.py @@ -2,9 +2,9 @@ TODO: Modify unittest doc. """ -import unittest import random import sys +import unittest from monty.string import remove_non_ascii, unicode2str diff --git a/tests/test_tempfile.py b/tests/test_tempfile.py index 34040356..51c9aff4 100644 --- a/tests/test_tempfile.py +++ b/tests/test_tempfile.py @@ -1,6 +1,6 @@ -import unittest -import shutil import os +import shutil +import unittest from monty.tempfile import ScratchDir diff --git a/tests/test_termcolor.py b/tests/test_termcolor.py index 248fb5e9..ab527e75 100644 --- a/tests/test_termcolor.py +++ b/tests/test_termcolor.py @@ -2,17 +2,12 @@ TODO: Modify unittest doc. """ -import unittest import os import sys +import unittest -from monty.termcolor import ( - cprint, - cprint_map, - enable, - get_terminal_size, - stream_has_colours, -) +from monty.termcolor import (cprint, cprint_map, enable, get_terminal_size, + stream_has_colours) class FuncTest(unittest.TestCase): From 255a3108adb9eb1f74368ef3dd605f80d45448fa Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 12:57:46 -0800 Subject: [PATCH 10/28] Update reqs --- requirements-ci.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-ci.txt b/requirements-ci.txt index 88859a07..ba251906 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -15,3 +15,5 @@ flake8==4.0.1 pandas==1.4.0 black==22.1.0 pylint==2.12.2 +orjson==3.6.7 +types-orjson==3.6.2 From 23ab17e371af026a44f80c5b8bcff9bf23511d29 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:00:34 +0000 Subject: [PATCH 11/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements-ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index ca1294b3..9bcc823f 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -17,4 +17,3 @@ black==22.1.0 pylint==2.12.2 orjson==3.6.7 types-orjson==3.6.2 - From 1e1e5b28731e6b587745a43f0cb12a611aa4f69c Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:06:33 -0800 Subject: [PATCH 12/28] Linting --- monty/design_patterns.py | 1 - setup.py | 1 - tests/test_design_patterns.py | 1 - tests/test_inspect.py | 2 +- tests/test_json.py | 2 +- tests/test_os.py | 3 +-- 6 files changed, 3 insertions(+), 7 deletions(-) diff --git a/monty/design_patterns.py b/monty/design_patterns.py index 6d7b3874..0da52c50 100644 --- a/monty/design_patterns.py +++ b/monty/design_patterns.py @@ -116,4 +116,3 @@ def write(*args): # pylint: disable=E0211 Does nothing... :param args: """ - pass diff --git a/setup.py b/setup.py index f3dfaa86..c12e7a59 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import io import os from setuptools import find_packages, setup diff --git a/tests/test_design_patterns.py b/tests/test_design_patterns.py index 6a74ae65..2f7f93bf 100644 --- a/tests/test_design_patterns.py +++ b/tests/test_design_patterns.py @@ -1,4 +1,3 @@ -import pickle import unittest from monty.design_patterns import cached_class, singleton diff --git a/tests/test_inspect.py b/tests/test_inspect.py index f9af8af2..55079f95 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -1,6 +1,6 @@ import unittest -from monty.inspect import * +from monty.inspect import all_subclasses, caller_name, find_top_pyfile class LittleCatA: diff --git a/tests/test_json.py b/tests/test_json.py index a920c5e5..c13735b9 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -148,7 +148,7 @@ def test_to_from_dict(self): self.assertRaises(NotImplementedError, obj.as_dict) obj = self.auto_mson(2, 3) d = obj.as_dict() - objd = self.auto_mson.from_dict(d) + self.auto_mson.from_dict(d) def test_unsafe_hash(self): GMC = GoodMSONClass diff --git a/tests/test_os.py b/tests/test_os.py index b680195d..5de9b852 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -1,9 +1,8 @@ import os -import platform import unittest from monty.os import cd, makedirs_p -from monty.os.path import find_exts, which, zpath +from monty.os.path import find_exts, zpath test_dir = os.path.join(os.path.dirname(__file__), "test_files") From 2ebae05d2d39ce99a15f085e65ce4d8083c930a5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:11:27 -0800 Subject: [PATCH 13/28] More linting --- docs_rst/_themes/flask_theme_support.py | 15 ++++++++------- docs_rst/conf.py | 6 +++--- requirements-ci.txt | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs_rst/_themes/flask_theme_support.py b/docs_rst/_themes/flask_theme_support.py index 33f47449..d38c8b45 100755 --- a/docs_rst/_themes/flask_theme_support.py +++ b/docs_rst/_themes/flask_theme_support.py @@ -1,7 +1,8 @@ # flasky extensions. flasky pygments style based on tango style from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal +from pygments.token import (Comment, Error, Generic, Keyword, Literal, Name, + Number, Operator, Other, Punctuation, String, + Whitespace) class FlaskyStyle(Style): @@ -10,12 +11,12 @@ class FlaskyStyle(Style): styles = { # No corresponding class for the following: - #Text: "", # class: '' + # Text: "", # class: '' Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' + Error: "#a40000 border:#ef2929", # class: 'err' Other: "#000000", # class 'x' - Comment: "italic #8f5902", # class: 'c' + Comment: "italic #8f5902", # class: 'c' Comment.Preproc: "noitalic", # class: 'cp' Keyword: "bold #004461", # class: 'k' @@ -62,7 +63,7 @@ class FlaskyStyle(Style): String: "#4e9a06", # class: 's' String.Backtick: "#4e9a06", # class: 'sb' String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment + String.Doc: "italic #8f5902", # class: 'sd' - like a comment String.Double: "#4e9a06", # class: 's2' String.Escape: "#4e9a06", # class: 'se' String.Heredoc: "#4e9a06", # class: 'sh' @@ -74,7 +75,7 @@ class FlaskyStyle(Style): Generic: "#000000", # class: 'g' Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' + Generic.Emph: "italic #000000", # class: 'ge' Generic.Error: "#ef2929", # class: 'gr' Generic.Heading: "bold #000080", # class: 'gh' Generic.Inserted: "#00A000", # class: 'gi' diff --git a/docs_rst/conf.py b/docs_rst/conf.py index 1deefa7f..99f0a4ef 100644 --- a/docs_rst/conf.py +++ b/docs_rst/conf.py @@ -11,8 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -22,7 +22,7 @@ sys.path.insert(0, os.path.dirname('../monty')) sys.path.insert(0, os.path.dirname('../..')) -from monty import __version__, __author__ +from monty import __author__, __version__ # -- General configuration ----------------------------------------------------- @@ -181,7 +181,7 @@ # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { +latex_elements = { # type: ignore # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', diff --git a/requirements-ci.txt b/requirements-ci.txt index 9bcc823f..8986d8fc 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -17,3 +17,4 @@ black==22.1.0 pylint==2.12.2 orjson==3.6.7 types-orjson==3.6.2 +types-requests==2.27.11 From 4423912ec2df3ead7351ae76e37437f23ff7fa2b Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:21:36 -0800 Subject: [PATCH 14/28] Black + fix isort profile --- .pre-commit-config.yaml | 1 + monty/dev.py | 6 ++---- tests/test_collections.py | 3 +-- tests/test_functools.py | 11 +++++++++-- tests/test_io.py | 9 +++++++-- tests/test_json.py | 3 +-- tests/test_shutil.py | 11 +++++++++-- tests/test_termcolor.py | 9 +++++++-- 8 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 596e9925..3487f911 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: rev: 5.10.1 hooks: - id: isort + args: ["--profile", "black"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 diff --git a/monty/dev.py b/monty/dev.py index fbb94faf..4f31f4b5 100644 --- a/monty/dev.py +++ b/monty/dev.py @@ -139,8 +139,7 @@ def get_ncpus(): # jython try: - from java.lang import \ - Runtime # pylint: disable=import-outside-toplevel + from java.lang import Runtime # pylint: disable=import-outside-toplevel runtime = Runtime.getRuntime() res = runtime.availableProcessors() @@ -214,8 +213,7 @@ def install_excepthook(hook_type="color", **kwargs): 0 if hook is installed successfully. """ try: - from IPython.core import \ - ultratb # pylint: disable=import-outside-toplevel + from IPython.core import ultratb # pylint: disable=import-outside-toplevel except ImportError: warnings.warn("Cannot install excepthook, IPyhon.core.ultratb not available") return 1 diff --git a/tests/test_collections.py b/tests/test_collections.py index 6469071d..87de9e7e 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,8 +1,7 @@ import os import unittest -from monty.collections import (AttrDict, FrozenAttrDict, Namespace, frozendict, - tree) +from monty.collections import AttrDict, FrozenAttrDict, Namespace, frozendict, tree test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_functools.py b/tests/test_functools.py index 19d47278..1b3b1c8c 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -3,8 +3,15 @@ import time import unittest -from monty.functools import (TimeoutError, lazy_property, lru_cache, prof_main, - return_if_raise, return_none_if_raise, timeout) +from monty.functools import ( + TimeoutError, + lazy_property, + lru_cache, + prof_main, + return_if_raise, + return_none_if_raise, + timeout, +) class TestLRUCache(unittest.TestCase): diff --git a/tests/test_io.py b/tests/test_io.py index 7494772c..973388a5 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,8 +6,13 @@ except ImportError: Path = None # type: ignore -from monty.io import (FileLock, FileLockException, reverse_readfile, - reverse_readline, zopen) +from monty.io import ( + FileLock, + FileLockException, + reverse_readfile, + reverse_readline, + zopen, +) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_json.py b/tests/test_json.py index c13735b9..4681e110 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -10,8 +10,7 @@ import pandas as pd from bson.objectid import ObjectId -from monty.json import (MontyDecoder, MontyEncoder, MSONable, _load_redirect, - jsanitize) +from monty.json import MontyDecoder, MontyEncoder, MSONable, _load_redirect, jsanitize from . import __version__ as tests_version diff --git a/tests/test_shutil.py b/tests/test_shutil.py index 2a79e94e..ab4c99a3 100644 --- a/tests/test_shutil.py +++ b/tests/test_shutil.py @@ -5,8 +5,15 @@ import unittest from gzip import GzipFile -from monty.shutil import (compress_dir, compress_file, copy_r, decompress_dir, - decompress_file, gzip_dir, remove) +from monty.shutil import ( + compress_dir, + compress_file, + copy_r, + decompress_dir, + decompress_file, + gzip_dir, + remove, +) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_termcolor.py b/tests/test_termcolor.py index ab527e75..9b9b7dae 100644 --- a/tests/test_termcolor.py +++ b/tests/test_termcolor.py @@ -6,8 +6,13 @@ import sys import unittest -from monty.termcolor import (cprint, cprint_map, enable, get_terminal_size, - stream_has_colours) +from monty.termcolor import ( + cprint, + cprint_map, + enable, + get_terminal_size, + stream_has_colours, +) class FuncTest(unittest.TestCase): From bf5ddaf119dbd4932477fd49a2c9654fc0c1b750 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:35:32 -0800 Subject: [PATCH 15/28] Fix docstring --- monty/json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monty/json.py b/monty/json.py index 07807360..3b6258dd 100644 --- a/monty/json.py +++ b/monty/json.py @@ -277,10 +277,10 @@ def encode(self, o): def handle_iters(self, o): """ - Recursive function for handling encoding of iterables + Recursive function for handling encoding of certain iterables Args: - o: Iterable Python object. + o: List, Dict, or Tuple object. Return: list of Python dict representations. From 372fe979729e4e7517f928cf22b1d6e93e9ed573 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:41:58 -0800 Subject: [PATCH 16/28] Fix pylint --- monty/json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monty/json.py b/monty/json.py index 3b6258dd..daa0599e 100644 --- a/monty/json.py +++ b/monty/json.py @@ -272,7 +272,7 @@ def encode(self, o): else: d = self.process(o) - e = orjson.dumps(d).decode("utf-8") + e = orjson.dumps(d).decode("utf-8") # pylint: disable=E1101 return e def handle_iters(self, o): @@ -492,7 +492,7 @@ def decode(self, s): :param s: string :return: Object. """ - d = orjson.loads(s) + d = orjson.loads(s) # pylint: disable=E1101 return self.process_decoded(d) @@ -590,8 +590,8 @@ def _serialize_callable(o): new_bound = MontyEncoder().process(bound) if new_bound == bound: raise TypeError("Only bound methods of classes or MSONable instances are supported.") - else: - bound = new_bound + + bound = new_bound return { "@module": o.__module__, From 6678b16842cf821826589e9c691912245a7f5a5a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 13:59:12 -0800 Subject: [PATCH 17/28] Default to regular json if orjson not present --- monty/json.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/monty/json.py b/monty/json.py index daa0599e..0bdbbd4d 100644 --- a/monty/json.py +++ b/monty/json.py @@ -40,6 +40,11 @@ except ImportError: YAML = None # type: ignore +try: + import orjson +except ImportError: + orjson = None # type: ignore + __version__ = "3.0.0" @@ -272,7 +277,11 @@ def encode(self, o): else: d = self.process(o) - e = orjson.dumps(d).decode("utf-8") # pylint: disable=E1101 + if orjson is not None: + e = orjson.dumps(d).decode("utf-8") # pylint: disable=E1101 + else: + e = json.dumps(d) + return e def handle_iters(self, o): @@ -492,7 +501,10 @@ def decode(self, s): :param s: string :return: Object. """ - d = orjson.loads(s) # pylint: disable=E1101 + if orjson is not None: + d = orjson.loads(s) # pylint: disable=E1101 + else: + d = json.loads(s) return self.process_decoded(d) From 5608cceada865553f43b430c845684a7099df6f3 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 8 Mar 2022 14:58:33 -0800 Subject: [PATCH 18/28] Remove duplicate import --- monty/json.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monty/json.py b/monty/json.py index 0bdbbd4d..7a609341 100644 --- a/monty/json.py +++ b/monty/json.py @@ -13,8 +13,6 @@ from inspect import getfullargspec from uuid import UUID -import orjson - try: import numpy as np except ImportError: From 5bfa829c4732a94a5233ef843bbf7838d388f8a3 Mon Sep 17 00:00:00 2001 From: munrojm Date: Tue, 8 Mar 2022 22:06:36 -0800 Subject: [PATCH 19/28] Fix recursive encoding --- monty/json.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/monty/json.py b/monty/json.py index 7a609341..66a6e980 100644 --- a/monty/json.py +++ b/monty/json.py @@ -12,6 +12,7 @@ from importlib import import_module from inspect import getfullargspec from uuid import UUID +import copy try: import numpy as np @@ -270,10 +271,11 @@ def encode(self, o): Return: JSON string representation. """ - if isinstance(o, (list, tuple, dict)): - d = self.handle_iters(o) - else: - d = self.process(o) + + d = self.process(o) + + if isinstance(d, (list, tuple, dict)): + self.handle_iters(d) if orjson is not None: e = orjson.dumps(d).decode("utf-8") # pylint: disable=E1101 @@ -292,21 +294,18 @@ def handle_iters(self, o): Return: list of Python dict representations. """ - d_list = {} if isinstance(o, dict) else [] - for item in o: - val = o[item] if isinstance(d_list, dict) else item - try: - d = self.process(val) - except TypeError: - d = self.handle_iters(val) + for ind, item in enumerate(o): + val = o[item] if isinstance(o, dict) else item + + d = self.process(val) + if isinstance(d, (list, tuple, dict)): + self.handle_iters(d) - if isinstance(d_list, dict): - d_list[item] = d + if isinstance(o, dict): + o[item] = d else: - d_list.append(d) - - return d_list + o[ind] = d def process(self, o) -> dict: # pylint: disable=E0202 """ @@ -384,11 +383,9 @@ def process(self, o) -> dict: # pylint: disable=E0202 except (AttributeError, ImportError): d["@version"] = None - d_p = self.handle_iters(d) - - return d_p + return d except AttributeError: - return o + return copy.deepcopy(o) class MontyDecoder(json.JSONDecoder): From 15dc92a08bd9ce7522bacd848912ae1e0ebb51bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 06:06:47 +0000 Subject: [PATCH 20/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monty/json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monty/json.py b/monty/json.py index 66a6e980..da7afcf8 100644 --- a/monty/json.py +++ b/monty/json.py @@ -2,6 +2,7 @@ JSON serialization and deserialization utilities. """ +import copy import datetime import json import os @@ -12,7 +13,6 @@ from importlib import import_module from inspect import getfullargspec from uuid import UUID -import copy try: import numpy as np @@ -271,9 +271,9 @@ def encode(self, o): Return: JSON string representation. """ - + d = self.process(o) - + if isinstance(d, (list, tuple, dict)): self.handle_iters(d) @@ -297,7 +297,7 @@ def handle_iters(self, o): for ind, item in enumerate(o): val = o[item] if isinstance(o, dict) else item - + d = self.process(val) if isinstance(d, (list, tuple, dict)): self.handle_iters(d) From 8548e1fd03250180b288139ad02855c529725d2e Mon Sep 17 00:00:00 2001 From: munrojm Date: Tue, 8 Mar 2022 22:24:34 -0800 Subject: [PATCH 21/28] Remove copy retun in process --- monty/json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/json.py b/monty/json.py index 66a6e980..e1949bef 100644 --- a/monty/json.py +++ b/monty/json.py @@ -385,7 +385,7 @@ def process(self, o) -> dict: # pylint: disable=E0202 return d except AttributeError: - return copy.deepcopy(o) + return o class MontyDecoder(json.JSONDecoder): From 5a5262d394ae419a6d567950a207166d5aafc75e Mon Sep 17 00:00:00 2001 From: munrojm Date: Tue, 8 Mar 2022 22:30:20 -0800 Subject: [PATCH 22/28] Remove copy import --- monty/json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monty/json.py b/monty/json.py index 41652f2a..5bea4d04 100644 --- a/monty/json.py +++ b/monty/json.py @@ -2,7 +2,6 @@ JSON serialization and deserialization utilities. """ -import copy import datetime import json import os From 106ffc2cac84e3a2effb25af72ef68ec613907f1 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 9 Mar 2022 15:43:43 -0800 Subject: [PATCH 23/28] Increase nesting on some tests --- monty/json.py | 1 + tests/test_json.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/monty/json.py b/monty/json.py index 5bea4d04..e4ecda6b 100644 --- a/monty/json.py +++ b/monty/json.py @@ -298,6 +298,7 @@ def handle_iters(self, o): val = o[item] if isinstance(o, dict) else item d = self.process(val) + if isinstance(d, (list, tuple, dict)): self.handle_iters(d) diff --git a/tests/test_json.py b/tests/test_json.py index 4681e110..a5295e86 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -102,6 +102,11 @@ def __init__(self, s): self.s = s +class ClassContainingNumpyArray(MSONable): + def __init__(self, np_a): + self.np_a = np_a + + class MSONableTest(unittest.TestCase): def setUp(self): self.good_cls = GoodMSONClass @@ -344,6 +349,22 @@ def test_numpy(self): d = jsanitize(x, strict=True) assert type(d["energies"][0]) == float + # Test data nested in a class + x = np.array([[1 + 1j, 2 + 1j], [3 + 1j, 4 + 1j]], dtype="complex64") + cls = ClassContainingNumpyArray(np_a={"a": [{"b": x}]}) + + d = json.loads(json.dumps(cls, cls=MontyEncoder)) + + self.assertEqual(d["np_a"]["a"][0]["b"]["@module"], "numpy") + self.assertEqual(d["np_a"]["a"][0]["b"]["@class"], "array") + self.assertEqual(d["np_a"]["a"][0]["b"]["data"], [[[1.0, 2.0], [3.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]]) + self.assertEqual(d["np_a"]["a"][0]["b"]["dtype"], "complex64") + + obj = ClassContainingNumpyArray.from_dict(d) + self.assertIsInstance(obj, ClassContainingNumpyArray) + self.assertIsInstance(obj.np_a["a"][0]["b"], np.ndarray) + self.assertEqual(obj.np_a["a"][0]["b"][0][1], 2 + 1j) + def test_pandas(self): cls = ClassContainingDataFrame(df=pd.DataFrame([{"a": 1, "b": 1}, {"a": 1, "b": 2}])) @@ -370,6 +391,18 @@ def test_pandas(self): self.assertIsInstance(obj.s, pd.Series) self.assertEqual(list(obj.s.a), [1, 2, 3]) + cls = ClassContainingSeries(s={"df": [pd.Series({"a": [1, 2, 3], "b": [4, 5, 6]})]}) + + d = json.loads(MontyEncoder().encode(cls)) + + self.assertEqual(d["s"]["df"][0]["@module"], "pandas") + self.assertEqual(d["s"]["df"][0]["@class"], "Series") + + obj = ClassContainingSeries.from_dict(d) + self.assertIsInstance(obj, ClassContainingSeries) + self.assertIsInstance(obj.s["df"][0], pd.Series) + self.assertEqual(list(obj.s["df"][0].a), [1, 2, 3]) + def test_callable(self): instance = MethodSerializationClass(a=1) for function in [ From c88e552deca22443b9241613991dffe03fe2091f Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 9 Mar 2022 16:00:58 -0800 Subject: [PATCH 24/28] Revert MontyEncoder changes --- monty/json.py | 58 +++------------------------------------------------ 1 file changed, 3 insertions(+), 55 deletions(-) diff --git a/monty/json.py b/monty/json.py index e4ecda6b..a5131373 100644 --- a/monty/json.py +++ b/monty/json.py @@ -253,71 +253,20 @@ class MontyEncoder(json.JSONEncoder): """ A Json Encoder which supports the MSONable API, plus adds support for numpy arrays, datetime objects, bson ObjectIds (requires bson). - Usage:: - # Add it as a *cls* keyword when using json.dump json.dumps(object, cls=MontyEncoder) """ - def encode(self, o): - """ - Overrides encode method on default JSONEncoder. - - Args: - o: Python object. - - Return: - JSON string representation. - """ - - d = self.process(o) - - if isinstance(d, (list, tuple, dict)): - self.handle_iters(d) - - if orjson is not None: - e = orjson.dumps(d).decode("utf-8") # pylint: disable=E1101 - else: - e = json.dumps(d) - - return e - - def handle_iters(self, o): + def default(self, o) -> dict: # pylint: disable=E0202 """ - Recursive function for handling encoding of certain iterables - - Args: - o: List, Dict, or Tuple object. - - Return: - list of Python dict representations. - """ - - for ind, item in enumerate(o): - val = o[item] if isinstance(o, dict) else item - - d = self.process(val) - - if isinstance(d, (list, tuple, dict)): - self.handle_iters(d) - - if isinstance(o, dict): - o[item] = d - else: - o[ind] = d - - def process(self, o) -> dict: # pylint: disable=E0202 - """ - Processing for sanitization before encoding. This method does two + Overriding default method for JSON encoding. This method does two things: (a) If an object has a to_dict property, return the to_dict output. (b) If the @module and @class keys are not in the to_dict, add them to the output automatically. If the object has no to_dict property, the default Python json encoder default method is called. - Args: o: Python object. - Return: Python dict representation. """ @@ -382,10 +331,9 @@ def process(self, o) -> dict: # pylint: disable=E0202 d["@version"] = str(module_version) except (AttributeError, ImportError): d["@version"] = None - return d except AttributeError: - return o + return json.JSONEncoder.default(self, o) class MontyDecoder(json.JSONDecoder): From c7960338d1b41b700629fd2d3f90f3156a07e1d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Mar 2022 00:01:54 +0000 Subject: [PATCH 25/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_io.py | 8 +------- tests/test_shutil.py | 10 +--------- tests/test_termcolor.py | 8 +------- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index 973388a5..cfec71df 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,13 +6,7 @@ except ImportError: Path = None # type: ignore -from monty.io import ( - FileLock, - FileLockException, - reverse_readfile, - reverse_readline, - zopen, -) +from monty.io import FileLock, FileLockException, reverse_readfile, reverse_readline, zopen test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_shutil.py b/tests/test_shutil.py index ab4c99a3..27ef7cd5 100644 --- a/tests/test_shutil.py +++ b/tests/test_shutil.py @@ -5,15 +5,7 @@ import unittest from gzip import GzipFile -from monty.shutil import ( - compress_dir, - compress_file, - copy_r, - decompress_dir, - decompress_file, - gzip_dir, - remove, -) +from monty.shutil import compress_dir, compress_file, copy_r, decompress_dir, decompress_file, gzip_dir, remove test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_termcolor.py b/tests/test_termcolor.py index 9b9b7dae..287c9b6e 100644 --- a/tests/test_termcolor.py +++ b/tests/test_termcolor.py @@ -6,13 +6,7 @@ import sys import unittest -from monty.termcolor import ( - cprint, - cprint_map, - enable, - get_terminal_size, - stream_has_colours, -) +from monty.termcolor import cprint, cprint_map, enable, get_terminal_size, stream_has_colours class FuncTest(unittest.TestCase): From b5438e178a52076f53685f628ae37b7d9b97a383 Mon Sep 17 00:00:00 2001 From: munrojm Date: Fri, 11 Mar 2022 12:13:04 -0800 Subject: [PATCH 26/28] Fix revert and pre-commit --- .pre-commit-config.yaml | 1 + monty/json.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c522043..4a2cd7a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,6 +38,7 @@ repos: rev: 5.10.1 hooks: - id: isort + args: ["--profile", "black"] - repo: https://github.com/psf/black rev: 22.1.0 diff --git a/monty/json.py b/monty/json.py index 0bd380c0..ace0f73e 100644 --- a/monty/json.py +++ b/monty/json.py @@ -555,13 +555,11 @@ def _serialize_callable(o): # we are only able to serialize bound methods if the object the method is # bound to is itself serializable if bound is not None: - - new_bound = MontyEncoder().process(bound) - if new_bound == bound: + try: + bound = MontyEncoder().default(bound) + except TypeError: raise TypeError("Only bound methods of classes or MSONable instances are supported.") - bound = new_bound - return { "@module": o.__module__, "@callable": getattr(o, "__qualname__", o.__name__), From 131e2c8e6bb2c9f6ace075aa48570f3c183540f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Mar 2022 20:13:15 +0000 Subject: [PATCH 27/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_io.py | 8 +++++++- tests/test_shutil.py | 10 +++++++++- tests/test_termcolor.py | 8 +++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test_io.py b/tests/test_io.py index cfec71df..973388a5 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,7 +6,13 @@ except ImportError: Path = None # type: ignore -from monty.io import FileLock, FileLockException, reverse_readfile, reverse_readline, zopen +from monty.io import ( + FileLock, + FileLockException, + reverse_readfile, + reverse_readline, + zopen, +) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_shutil.py b/tests/test_shutil.py index 27ef7cd5..ab4c99a3 100644 --- a/tests/test_shutil.py +++ b/tests/test_shutil.py @@ -5,7 +5,15 @@ import unittest from gzip import GzipFile -from monty.shutil import compress_dir, compress_file, copy_r, decompress_dir, decompress_file, gzip_dir, remove +from monty.shutil import ( + compress_dir, + compress_file, + copy_r, + decompress_dir, + decompress_file, + gzip_dir, + remove, +) test_dir = os.path.join(os.path.dirname(__file__), "test_files") diff --git a/tests/test_termcolor.py b/tests/test_termcolor.py index 287c9b6e..9b9b7dae 100644 --- a/tests/test_termcolor.py +++ b/tests/test_termcolor.py @@ -6,7 +6,13 @@ import sys import unittest -from monty.termcolor import cprint, cprint_map, enable, get_terminal_size, stream_has_colours +from monty.termcolor import ( + cprint, + cprint_map, + enable, + get_terminal_size, + stream_has_colours, +) class FuncTest(unittest.TestCase): From a01a6aa8a9574892a21dfed3f387d85ef189d89b Mon Sep 17 00:00:00 2001 From: munrojm Date: Fri, 11 Mar 2022 12:36:06 -0800 Subject: [PATCH 28/28] Include imports with pylint --- pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index b0e764fd..ab547443 100644 --- a/pylintrc +++ b/pylintrc @@ -368,7 +368,7 @@ ignore-comments=yes ignore-docstrings=yes # Ignore imports when computing similarities. -ignore-imports=yes +ignore-imports=no # Minimum lines number of a similarity. min-similarity-lines=4