From 5a53459099ad5fd309bf9ae2d9850711d0eebeb0 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Tue, 26 Jul 2022 11:50:39 +0200 Subject: [PATCH 01/22] WIP: add support for de/serializing objects with json, msgpack or pickle --- .gitignore | 2 + setup.cfg | 6 ++ src/ufoLib2/converters.py | 8 +++ src/ufoLib2/objects/anchor.py | 2 + src/ufoLib2/objects/component.py | 2 + src/ufoLib2/objects/contour.py | 2 + src/ufoLib2/objects/dataSet.py | 2 + src/ufoLib2/objects/features.py | 3 + src/ufoLib2/objects/font.py | 2 + src/ufoLib2/objects/glyph.py | 2 + src/ufoLib2/objects/guideline.py | 2 + src/ufoLib2/objects/image.py | 3 + src/ufoLib2/objects/imageSet.py | 2 + src/ufoLib2/objects/kerning.py | 3 + src/ufoLib2/objects/layer.py | 2 + src/ufoLib2/objects/layerSet.py | 2 + src/ufoLib2/objects/lib.py | 2 + src/ufoLib2/objects/point.py | 3 + src/ufoLib2/serde/__init__.py | 111 +++++++++++++++++++++++++++++++ src/ufoLib2/serde/json.py | 43 ++++++++++++ src/ufoLib2/serde/msgpack.py | 19 ++++++ src/ufoLib2/serde/pickle.py | 16 +++++ 22 files changed, 239 insertions(+) create mode 100644 src/ufoLib2/serde/__init__.py create mode 100644 src/ufoLib2/serde/json.py create mode 100644 src/ufoLib2/serde/msgpack.py create mode 100644 src/ufoLib2/serde/pickle.py diff --git a/.gitignore b/.gitignore index 7016c34f..095ca33a 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,8 @@ venv/ ENV/ env.bak/ venv.bak/ +venv-* +.venv-* # Spyder project settings .spyderproject diff --git a/setup.cfg b/setup.cfg index 58634bd7..99a7c881 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,12 @@ install_requires = [options.extras_require] lxml = lxml converters = cattrs >= 1.10.0 +json = + cattrs >= 1.1.0 + orjson ; platform_python_implementation != 'PyPy' +msgpack = + cattrs >= 1.1.0 + msgpack [options.packages.find] where = src diff --git a/src/ufoLib2/converters.py b/src/ufoLib2/converters.py index a8b766c7..f6e84495 100644 --- a/src/ufoLib2/converters.py +++ b/src/ufoLib2/converters.py @@ -143,3 +143,11 @@ def structure_bytes(v: str, _: Any) -> bytes: structure = default_converter.structure unstructure = default_converter.unstructure + +# same as default_converter but allows bytes +binary_converter = GenConverter( + omit_if_default=True, + forbid_extra_keys=True, + prefer_attrib_converters=False, +) +register_hooks(binary_converter, allow_bytes=True) diff --git a/src/ufoLib2/objects/anchor.py b/src/ufoLib2/objects/anchor.py index 801115aa..b31d67d0 100644 --- a/src/ufoLib2/objects/anchor.py +++ b/src/ufoLib2/objects/anchor.py @@ -5,8 +5,10 @@ from attr import define from ufoLib2.objects.misc import AttrDictMixin +from ufoLib2.serde import serde +@serde @define class Anchor(AttrDictMixin): """Represents a single anchor. diff --git a/src/ufoLib2/objects/component.py b/src/ufoLib2/objects/component.py index e2fa90db..b67945c7 100644 --- a/src/ufoLib2/objects/component.py +++ b/src/ufoLib2/objects/component.py @@ -9,11 +9,13 @@ from fontTools.pens.pointPen import AbstractPointPen, PointToSegmentPen from ufoLib2.objects.misc import BoundingBox +from ufoLib2.serde import serde from ufoLib2.typing import GlyphSet from .misc import _convert_transform, getBounds, getControlBounds +@serde @define class Component: """Represents a reference to another glyph in the same layer. diff --git a/src/ufoLib2/objects/contour.py b/src/ufoLib2/objects/contour.py index d5dcb84e..a53e971e 100644 --- a/src/ufoLib2/objects/contour.py +++ b/src/ufoLib2/objects/contour.py @@ -10,6 +10,7 @@ from ufoLib2.objects.misc import BoundingBox, getBounds, getControlBounds from ufoLib2.objects.point import Point +from ufoLib2.serde import serde from ufoLib2.typing import GlyphSet # For Python 3.7 compatibility. @@ -19,6 +20,7 @@ ContourMapping = MutableSequence +@serde @define class Contour(ContourMapping): """Represents a contour as a list of points. diff --git a/src/ufoLib2/objects/dataSet.py b/src/ufoLib2/objects/dataSet.py index f1a3e9c2..191321aa 100644 --- a/src/ufoLib2/objects/dataSet.py +++ b/src/ufoLib2/objects/dataSet.py @@ -3,8 +3,10 @@ from fontTools.ufoLib import UFOReader, UFOWriter from ufoLib2.objects.misc import DataStore +from ufoLib2.serde import serde +@serde class DataSet(DataStore): """Represents a mapping of POSIX filename strings to arbitrary data bytes. diff --git a/src/ufoLib2/objects/features.py b/src/ufoLib2/objects/features.py index fddc2aeb..265eebbf 100644 --- a/src/ufoLib2/objects/features.py +++ b/src/ufoLib2/objects/features.py @@ -5,6 +5,8 @@ from attr import define +from ufoLib2.serde import serde + if TYPE_CHECKING: from cattr import GenConverter @@ -12,6 +14,7 @@ RE_NEWLINES = re.compile(r"\r\n|\r") +@serde @define class Features: """A data class representing UFO features. diff --git a/src/ufoLib2/objects/font.py b/src/ufoLib2/objects/font.py index e6fa145a..2ec07743 100644 --- a/src/ufoLib2/objects/font.py +++ b/src/ufoLib2/objects/font.py @@ -38,6 +38,7 @@ _object_lib, _prune_object_libs, ) +from ufoLib2.serde import serde from ufoLib2.typing import HasIdentifier, PathLike, T @@ -73,6 +74,7 @@ def _set_kerning(self: Font, value: Mapping[KerningPair, float]) -> None: self._kerning = _convert_Kerning(value) +@serde @define(kw_only=True) class Font: """A data class representing a single Unified Font Object (UFO). diff --git a/src/ufoLib2/objects/glyph.py b/src/ufoLib2/objects/glyph.py index b90eb5be..d87ea0a4 100644 --- a/src/ufoLib2/objects/glyph.py +++ b/src/ufoLib2/objects/glyph.py @@ -20,9 +20,11 @@ from ufoLib2.objects.lib import Lib, _convert_Lib, _get_lib, _set_lib from ufoLib2.objects.misc import BoundingBox, _object_lib, getBounds, getControlBounds from ufoLib2.pointPens.glyphPointPen import GlyphPointPen +from ufoLib2.serde import serde from ufoLib2.typing import GlyphSet, HasIdentifier +@serde @define class Glyph: """Represents a glyph, containing contours, components, anchors and various diff --git a/src/ufoLib2/objects/guideline.py b/src/ufoLib2/objects/guideline.py index 08e9d190..75dd434e 100644 --- a/src/ufoLib2/objects/guideline.py +++ b/src/ufoLib2/objects/guideline.py @@ -5,8 +5,10 @@ from attr import define from ufoLib2.objects.misc import AttrDictMixin +from ufoLib2.serde import serde +@serde @define class Guideline(AttrDictMixin): """Represents a single guideline. diff --git a/src/ufoLib2/objects/image.py b/src/ufoLib2/objects/image.py index 7d2c53ef..2aa08212 100644 --- a/src/ufoLib2/objects/image.py +++ b/src/ufoLib2/objects/image.py @@ -6,6 +6,8 @@ from attr import define, field from fontTools.misc.transform import Identity, Transform +from ufoLib2.serde import serde + from .misc import _convert_transform # For Python 3.7 compatibility. @@ -15,6 +17,7 @@ ImageMapping = Mapping +@serde @define class Image(ImageMapping): """Represents a background image reference. diff --git a/src/ufoLib2/objects/imageSet.py b/src/ufoLib2/objects/imageSet.py index ffc1380b..66985cc3 100644 --- a/src/ufoLib2/objects/imageSet.py +++ b/src/ufoLib2/objects/imageSet.py @@ -3,8 +3,10 @@ from fontTools.ufoLib import UFOReader, UFOWriter from ufoLib2.objects.misc import DataStore +from ufoLib2.serde import serde +@serde class ImageSet(DataStore): """Represents a mapping of POSIX filename strings to arbitrary image data. diff --git a/src/ufoLib2/objects/kerning.py b/src/ufoLib2/objects/kerning.py index b40a099e..1e14d43c 100644 --- a/src/ufoLib2/objects/kerning.py +++ b/src/ufoLib2/objects/kerning.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING, Dict, Mapping, Tuple +from ufoLib2.serde import serde + if TYPE_CHECKING: from typing import Type @@ -10,6 +12,7 @@ KerningPair = Tuple[str, str] +@serde class Kerning(Dict[KerningPair, float]): def as_nested_dicts(self) -> dict[str, dict[str, float]]: result: dict[str, dict[str, float]] = {} diff --git a/src/ufoLib2/objects/layer.py b/src/ufoLib2/objects/layer.py index 670dcdf7..39a85b67 100644 --- a/src/ufoLib2/objects/layer.py +++ b/src/ufoLib2/objects/layer.py @@ -24,6 +24,7 @@ _prune_object_libs, unionBounds, ) +from ufoLib2.serde import serde from ufoLib2.typing import T if TYPE_CHECKING: @@ -64,6 +65,7 @@ def _convert_glyphs(value: dict[str, Glyph] | Sequence[Glyph]) -> dict[str, Glyp return result +@serde @define class Layer: """Represents a Layer that holds Glyph objects. diff --git a/src/ufoLib2/objects/layerSet.py b/src/ufoLib2/objects/layerSet.py index 85e03867..b649f707 100644 --- a/src/ufoLib2/objects/layerSet.py +++ b/src/ufoLib2/objects/layerSet.py @@ -18,6 +18,7 @@ from ufoLib2.errors import Error from ufoLib2.objects.layer import Layer from ufoLib2.objects.misc import _deepcopy_unlazify_attrs +from ufoLib2.serde import serde from ufoLib2.typing import T if TYPE_CHECKING: @@ -33,6 +34,7 @@ def _must_have_at_least_one_item(self: Any, attribute: Any, value: Sized) -> Non raise ValueError("value must have at least one item.") +@serde @define class LayerSet: """Represents a mapping of layer names to Layer objects. diff --git a/src/ufoLib2/objects/lib.py b/src/ufoLib2/objects/lib.py index 4705851a..769d8d31 100644 --- a/src/ufoLib2/objects/lib.py +++ b/src/ufoLib2/objects/lib.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, Mapping, Union, cast from ufoLib2.constants import DATA_LIB_KEY +from ufoLib2.serde import serde if TYPE_CHECKING: from typing import Type @@ -67,6 +68,7 @@ def _structure_data_inplace( _structure_data_inplace(k, v, value, converter) +@serde class Lib(Dict[str, Any]): def _unstructure(self, converter: GenConverter) -> dict[str, Any]: # avoid encoding if converter supports bytes natively diff --git a/src/ufoLib2/objects/point.py b/src/ufoLib2/objects/point.py index fcfb2fd6..e9ffcf9f 100644 --- a/src/ufoLib2/objects/point.py +++ b/src/ufoLib2/objects/point.py @@ -4,7 +4,10 @@ from attr import define +from ufoLib2.serde import serde + +@serde @define class Point: """Represents a single point. diff --git a/src/ufoLib2/serde/__init__.py b/src/ufoLib2/serde/__init__.py new file mode 100644 index 00000000..5356f39f --- /dev/null +++ b/src/ufoLib2/serde/__init__.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from functools import partialmethod +from importlib import import_module +from typing import IO, Any, AnyStr, BinaryIO, Callable, Type, cast + +from ufoLib2.typing import PathLike, T + +_SERDE_FORMATS_ = ("json", "msgpack", "pickle") + + +def _loads( + cls: Type[Any], s: str | bytes, *, __serde_submodule: Any, **kwargs: Any +) -> Any: + return __serde_submodule.loads(s, cls, **kwargs) + + +def _load( + cls: Type[Any], + fp: PathLike | IO[AnyStr], + *, + __loads_method: Callable[..., Any], + **kwargs: Any, +) -> Any: + data: str | bytes + if hasattr(fp, "read"): + fp = cast(IO[AnyStr], fp) + data = fp.read() + else: + fp = cast(PathLike, fp) + with open(fp, "rb") as f: + data = f.read() + return __loads_method(data, **kwargs) + + +def _dumps(self: Any, *, __serde_submodule: Any, **kwargs: Any) -> Any: + return __serde_submodule.dumps(self, **kwargs) + + +def _dump( + self: Any, fp: PathLike | BinaryIO, *, __dumps_method_name: str, **kwargs: Any +) -> None: + data: bytes = getattr(self, __dumps_method_name)(**kwargs) + if hasattr(fp, "write"): + fp = cast(BinaryIO, fp) + fp.write(data) + else: + fp = cast(PathLike, fp) + with open(fp, "wb") as f: + f.write(data) + + +def serde(cls: Type[T]) -> Type[T]: + """Decorator to add serialization support to a ufoLib2 class. + + Currently JSON, MessagePack (msgpack) and Pickle are the supported formats, + but other formats may be added in the future. + + Pickle works out of the box, whereas the others require additional extras + to be installed: e.g. ufoLib2[json,msgpack]. If required, this will install + the `cattrs` library for structuring/unstructuring custom objects from/to + serializable data structures (also available with ufoLib2[converters] extra). + + If any of the optional dependencies fails to be imported, this decorator will + raise an ImportError when any of the related methods are called. + + If the faster `orjson` library is present, it will be used in place of the + built-in `json` library. + """ + + supported_formats = [] + for fmt in _SERDE_FORMATS_: + + try: + serde_submodule = import_module(f"ufoLib2.serde.{fmt}") + except ImportError as e: + exc = e + + def raise_error(*args: Any, **kwargs: Any) -> None: + raise exc + + for method in ("loads", "load", "dumps", "dump"): + setattr(cls, f"{fmt}_{method}", raise_error) + else: + setattr( + cls, + f"{fmt}_loads", + partialmethod(classmethod(_loads), __serde_submodule=serde_submodule), + ) + setattr( + cls, + f"{fmt}_load", + partialmethod( + classmethod(_load), __loads_method=getattr(cls, f"{fmt}_loads") + ), + ) + setattr( + cls, + f"{fmt}_dumps", + partialmethod(_dumps, __serde_submodule=serde_submodule), + ) + setattr( + cls, + f"{fmt}_dump", + partialmethod(_dump, __dumps_method_name=f"{fmt}_dumps"), + ) + supported_formats.append(fmt) + + setattr(cls, "_SERDE_FORMATS_", tuple(supported_formats)) + + return cls diff --git a/src/ufoLib2/serde/json.py b/src/ufoLib2/serde/json.py new file mode 100644 index 00000000..b6e27565 --- /dev/null +++ b/src/ufoLib2/serde/json.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any, Type, cast + +from ufoLib2.converters import structure, unstructure +from ufoLib2.typing import T + +have_orjson = False +try: + import orjson as json # type: ignore + + have_orjson = True +except ImportError: + import json # type: ignore + + +def dumps( + obj: Any, + indent: int | None = None, + sort_keys: bool = False, + **kwargs: Any, +) -> bytes: + data = unstructure(obj) + + if have_orjson: + if indent is not None: + if indent != 2: + raise ValueError("indent must be 2 or None for orjson") + kwargs["option"] = kwargs.pop("option", 0) | json.OPT_INDENT_2 + if sort_keys: + kwargs["option"] = kwargs.pop("option", 0) | json.OPT_SORT_KEYS + # orjson.dumps always returns bytes + result = json.dumps(data, **kwargs) + else: + # built-in json.dumps returns a string, not bytes, hence the encoding + s = json.dumps(data, indent=indent, sort_keys=sort_keys, **kwargs) + result = s.encode("utf-8") + return cast(bytes, result) + + +def loads(s: str | bytes, object_class: Type[T], **kwargs: Any) -> T: + data = json.loads(s, **kwargs) + return structure(data, object_class) diff --git a/src/ufoLib2/serde/msgpack.py b/src/ufoLib2/serde/msgpack.py new file mode 100644 index 00000000..29951659 --- /dev/null +++ b/src/ufoLib2/serde/msgpack.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Any, Type, cast + +import msgpack # type: ignore + +from ufoLib2.converters import binary_converter +from ufoLib2.typing import T + + +def dumps(obj: Any, **kwargs: Any) -> bytes: + data = binary_converter.unstructure(obj) + result = msgpack.packb(data, **kwargs) + return cast(bytes, result) + + +def loads(s: bytes, object_class: Type[T], **kwargs: Any) -> T: + data = msgpack.unpackb(s, **kwargs) + return binary_converter.structure(data, object_class) diff --git a/src/ufoLib2/serde/pickle.py b/src/ufoLib2/serde/pickle.py new file mode 100644 index 00000000..40ee1bab --- /dev/null +++ b/src/ufoLib2/serde/pickle.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import pickle +from typing import Any, Type + +from ufoLib2.typing import T + + +def dumps(obj: Any, **kwargs: Any) -> bytes: + return pickle.dumps(obj, **kwargs) + + +def loads(s: bytes, object_class: Type[T], **kwargs: Any) -> T: + obj = pickle.loads(s, **kwargs) + assert isinstance(obj, object_class) + return obj From bc82a2ad7187f782ba6412458f2117a47737d519 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 12:57:29 +0200 Subject: [PATCH 02/22] LayerSet/Layer: record self._lazy and make unlazify idempotent similar to Font, DataSet, ImageSet --- src/ufoLib2/objects/layer.py | 8 ++++++-- src/ufoLib2/objects/layerSet.py | 8 ++++++-- src/ufoLib2/objects/misc.py | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/ufoLib2/objects/layer.py b/src/ufoLib2/objects/layer.py index 39a85b67..0695625b 100644 --- a/src/ufoLib2/objects/layer.py +++ b/src/ufoLib2/objects/layer.py @@ -117,6 +117,7 @@ class Layer: the default attribute is automatically True. Exactly one layer must be marked as default in a font.""" + _lazy: Optional[bool] = field(default=None, init=False, eq=False) _glyphSet: Any = field(default=None, init=False, eq=False) def __attrs_post_init__(self) -> None: @@ -148,6 +149,7 @@ def read( glyphSet.readGlyph(glyphName, glyph, glyph.getPointPen()) glyphs[glyphName] = glyph self = cls(name, glyphs, default=default) + self._lazy = lazy if lazy: self._glyphSet = glyphSet glyphSet.readLayerInfo(self) @@ -155,8 +157,10 @@ def read( def unlazify(self) -> None: """Load all glyphs into memory.""" - for _ in self: - pass + if self._lazy: + for _ in self: + pass + self._lazy = False __deepcopy__ = _deepcopy_unlazify_attrs diff --git a/src/ufoLib2/objects/layerSet.py b/src/ufoLib2/objects/layerSet.py index b649f707..c4c3c4c5 100644 --- a/src/ufoLib2/objects/layerSet.py +++ b/src/ufoLib2/objects/layerSet.py @@ -76,6 +76,7 @@ class LayerSet: _defaultLayer: Layer = field(default=_LAYER_NOT_LOADED, eq=False) + _lazy: Optional[bool] = field(default=None, init=False, eq=False) _reader: Optional[UFOReader] = field(default=None, init=False, eq=False) def __attrs_post_init__(self) -> None: @@ -167,6 +168,7 @@ def read(cls, reader: UFOReader, lazy: bool = True) -> LayerSet: assert defaultLayer is not None self = cls(layers=layers, defaultLayer=defaultLayer) + self._lazy = lazy if lazy: self._reader = reader @@ -174,8 +176,10 @@ def read(cls, reader: UFOReader, lazy: bool = True) -> LayerSet: def unlazify(self) -> None: """Load all layers into memory.""" - for layer in self: - layer.unlazify() + if self._lazy: + for layer in self: + layer.unlazify() + self._lazy = False __deepcopy__ = _deepcopy_unlazify_attrs diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index bb87fd6c..9eac6053 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -70,7 +70,7 @@ def unionBounds( def _deepcopy_unlazify_attrs(self: Any, memo: Any) -> Any: - if getattr(self, "_lazy", True) and hasattr(self, "unlazify"): + if self._lazy: self.unlazify() return self.__class__( **{ From c1322fe7e181abc2efb3a91155dd55740802be49 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 14:47:55 +0200 Subject: [PATCH 03/22] remove unused Attribute.metadata[copyable] we currently don't have any attributes that are init=True and copyable=False, the only copyable=False is Font.path which is also init=False, so simply checking if init=False is enough --- src/ufoLib2/objects/font.py | 4 +--- src/ufoLib2/objects/misc.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ufoLib2/objects/font.py b/src/ufoLib2/objects/font.py index 2ec07743..099e61cb 100644 --- a/src/ufoLib2/objects/font.py +++ b/src/ufoLib2/objects/font.py @@ -165,9 +165,7 @@ class Font: """ImageSet: A mapping of image file paths to arbitrary image data.""" # init=False args, set by alternate open/read classmethod constructors - _path: Optional[PathLike] = field( - default=None, metadata=dict(copyable=False), eq=False, init=False - ) + _path: Optional[PathLike] = field(default=None, eq=False, init=False) _lazy: Optional[bool] = field(default=None, init=False, eq=False) _reader: Optional[UFOReader] = field(default=None, init=False, eq=False) _fileStructure: Optional[UFOFileStructure] = field( diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index 9eac6053..d2bac47d 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -78,7 +78,7 @@ def _deepcopy_unlazify_attrs(self: Any, memo: Any) -> Any: getattr(self, a.name), memo ) for a in attr.fields(self.__class__) - if a.init and a.metadata.get("copyable", True) + if a.init }, ) From e0f3d6af1cd9be371c28296c398ab97e37036c06 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 14:49:03 +0200 Subject: [PATCH 04/22] make lazy Font/LayerSet/Layer/DataStore pickleable --- src/ufoLib2/objects/font.py | 5 +++++ src/ufoLib2/objects/layer.py | 5 +++++ src/ufoLib2/objects/layerSet.py | 9 ++++++++- src/ufoLib2/objects/misc.py | 25 +++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/ufoLib2/objects/font.py b/src/ufoLib2/objects/font.py index 099e61cb..03d530de 100644 --- a/src/ufoLib2/objects/font.py +++ b/src/ufoLib2/objects/font.py @@ -35,8 +35,10 @@ from ufoLib2.objects.misc import ( BoundingBox, _deepcopy_unlazify_attrs, + _getstate_unlazify_attrs, _object_lib, _prune_object_libs, + _setstate_attrs, ) from ufoLib2.serde import serde from ufoLib2.typing import HasIdentifier, PathLike, T @@ -318,6 +320,9 @@ def unlazify(self) -> None: __deepcopy__ = _deepcopy_unlazify_attrs + __getstate__ = _getstate_unlazify_attrs + __setstate__ = _setstate_attrs + @property def glyphOrder(self) -> list[str]: """The font's glyph order. diff --git a/src/ufoLib2/objects/layer.py b/src/ufoLib2/objects/layer.py index 0695625b..6d882dc2 100644 --- a/src/ufoLib2/objects/layer.py +++ b/src/ufoLib2/objects/layer.py @@ -21,7 +21,9 @@ from ufoLib2.objects.misc import ( BoundingBox, _deepcopy_unlazify_attrs, + _getstate_unlazify_attrs, _prune_object_libs, + _setstate_attrs, unionBounds, ) from ufoLib2.serde import serde @@ -164,6 +166,9 @@ def unlazify(self) -> None: __deepcopy__ = _deepcopy_unlazify_attrs + __getstate__ = _getstate_unlazify_attrs + __setstate__ = _setstate_attrs + def __contains__(self, name: object) -> bool: return name in self._glyphs diff --git a/src/ufoLib2/objects/layerSet.py b/src/ufoLib2/objects/layerSet.py index c4c3c4c5..654846c3 100644 --- a/src/ufoLib2/objects/layerSet.py +++ b/src/ufoLib2/objects/layerSet.py @@ -17,7 +17,11 @@ from ufoLib2.constants import DEFAULT_LAYER_NAME from ufoLib2.errors import Error from ufoLib2.objects.layer import Layer -from ufoLib2.objects.misc import _deepcopy_unlazify_attrs +from ufoLib2.objects.misc import ( + _deepcopy_unlazify_attrs, + _getstate_unlazify_attrs, + _setstate_attrs, +) from ufoLib2.serde import serde from ufoLib2.typing import T @@ -183,6 +187,9 @@ def unlazify(self) -> None: __deepcopy__ = _deepcopy_unlazify_attrs + __getstate__ = _getstate_unlazify_attrs + __setstate__ = _setstate_attrs + @staticmethod def _loadLayer( reader: UFOReader, layerName: str, lazy: bool = True, default: bool = False diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index d2bac47d..2248d46e 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -15,6 +15,7 @@ Optional, Sequence, Set, + Tuple, Type, TypeVar, cast, @@ -83,6 +84,27 @@ def _deepcopy_unlazify_attrs(self: Any, memo: Any) -> Any: ) +def _getstate_unlazify_attrs(self: Any) -> Tuple[Any, ...]: + if self._lazy: + self.unlazify() + return tuple( + getattr(self, a.name) if a.init else a.default + for a in attr.fields(self.__class__) + ) + + +_obj_setattr = object.__setattr__ + + +# Since we override __getstate__, we must also override __setstate__. +# Below is adapted from `attrs._make._ClassBuilder._make_getstate_setstate` method: +# https://github.com/python-attrs/attrs/blob/36ed0204/src/attr/_make.py#L931-L937 +def _setstate_attrs(self: Any, state: Tuple[Any, ...]) -> None: + _bound_setattr = _obj_setattr.__get__(self, attr.Attribute) # type: ignore + for a, v in zip(attr.fields(self.__class__), state): + _bound_setattr(a.name, v) + + def _object_lib(parent_lib: dict[str, Any], obj: HasIdentifier) -> dict[str, Any]: if obj.identifier is None: # Use UUID4 because it allows us to set a new identifier without @@ -223,6 +245,9 @@ def unlazify(self) -> None: __deepcopy__ = _deepcopy_unlazify_attrs + __getstate__ = _getstate_unlazify_attrs + __setstate__ = _setstate_attrs + # MutableMapping methods def __len__(self) -> int: From 829e35ac72920d4313208964b0b73a84edad4e92 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 17:06:38 +0200 Subject: [PATCH 05/22] recompile requirements.txt with new [json,msgpack] extras --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ba6fa44..96d5fb7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --extra=converters --extra=lxml setup.cfg +# pip-compile --extra=converters --extra=json --extra=lxml --extra=msgpack setup.cfg # appdirs==1.4.4 # via fs @@ -18,6 +18,10 @@ fs==2.4.15 # via fonttools lxml==4.9.1 # via ufoLib2 (setup.cfg) +msgpack==1.0.4 + # via ufoLib2 (setup.cfg) +orjson==3.7.8 ; platform_python_implementation != "PyPy" + # via ufoLib2 (setup.cfg) pytz==2021.3 # via fs six==1.16.0 From d3a2b6a2d51fcec501afdad9360c693327a3e4a9 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 18:57:04 +0200 Subject: [PATCH 06/22] add tests/ufoLib2/serde --- tests/serde/test_json.py | 89 +++++++++++++++++++++++++++++++++++++ tests/serde/test_msgpack.py | 51 +++++++++++++++++++++ tests/serde/test_pickle.py | 59 ++++++++++++++++++++++++ tests/serde/test_serde.py | 29 ++++++++++++ 4 files changed, 228 insertions(+) create mode 100644 tests/serde/test_json.py create mode 100644 tests/serde/test_msgpack.py create mode 100644 tests/serde/test_pickle.py create mode 100644 tests/serde/test_serde.py diff --git a/tests/serde/test_json.py b/tests/serde/test_json.py new file mode 100644 index 00000000..bcda8847 --- /dev/null +++ b/tests/serde/test_json.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import ufoLib2.objects +import ufoLib2.serde.json + + +@pytest.mark.parametrize("have_orjson", [False, True], ids=["no-orjson", "with-orjson"]) +def test_dumps_loads( + monkeypatch: Any, have_orjson: bool, ufo_UbuTestData: ufoLib2.objects.Font +) -> None: + if not have_orjson: + monkeypatch.setattr(ufoLib2.serde.json, "have_orjson", have_orjson) + monkeypatch.setattr(ufoLib2.serde.json, "json", json) + + font = ufo_UbuTestData + data = font.json_dumps() # type: ignore + + assert isinstance(data, bytes) + + if have_orjson: + # with default indent=0, orjson adds no space between keys and values + assert data[:21] == b'{"layers":[{"name":"p' + else: + # built-in json always adds space between keys and values + assert data[:21] == b'{"layers": [{"name": ' + + font2 = ufoLib2.objects.Font.json_loads(data) # type: ignore + + assert font == font2 + + +@pytest.mark.parametrize("have_orjson", [False, True], ids=["no-orjson", "with-orjson"]) +@pytest.mark.parametrize("indent", [None, 2], ids=["no-indent", "indent-2"]) +@pytest.mark.parametrize("sort_keys", [False, True], ids=["no-sort-keys", "sort-keys"]) +def test_dump_load( + monkeypatch: Any, + tmp_path: Path, + ufo_UbuTestData: ufoLib2.objects.Font, + have_orjson: bool, + indent: int | None, + sort_keys: bool, +) -> None: + if not have_orjson: + monkeypatch.setattr(ufoLib2.serde.json, "have_orjson", have_orjson) + monkeypatch.setattr(ufoLib2.serde.json, "json", json) + + font = ufo_UbuTestData + with open(tmp_path / "test.json", "wb") as f: + font.json_dump(f, indent=indent, sort_keys=sort_keys) # type: ignore + + with open(tmp_path / "test.json", "rb") as f: + font2 = ufoLib2.objects.Font.json_load(f) # type: ignore + + assert font == font2 + + +@pytest.mark.parametrize("indent", [1, 3], ids=["indent-1", "indent-3"]) +def test_indent_not_2_orjson(indent: int) -> None: + with pytest.raises(ValueError): + ufoLib2.serde.json.dumps(None, indent=indent) + + +def test_not_allow_bytes(ufo_UbuTestData: ufoLib2.objects.Font) -> None: + font = ufo_UbuTestData + + # DataSet values are binary data (bytes) + assert all(isinstance(v, bytes) for v in font.data.values()) + + # bytes are are not allowed in JSON, so our converter automatically + # translates them to Base64 strings upon serializig + s = font.data.json_dumps() # type: ignore + + # check that (before structuring the DataSet object) the json data + # contains in fact str, not bytes + raw_data = json.loads(s) + assert isinstance(raw_data, dict) + assert all(isinstance(v, str) for v in raw_data.values()) + + # check that (after structuring the DataSet object) the json data + # now contains bytes, like the original data + data = font.data.json_loads(s) # type: ignore + assert isinstance(data, ufoLib2.objects.DataSet) + assert all(isinstance(v, bytes) for v in data.values()) diff --git a/tests/serde/test_msgpack.py b/tests/serde/test_msgpack.py new file mode 100644 index 00000000..eeebf045 --- /dev/null +++ b/tests/serde/test_msgpack.py @@ -0,0 +1,51 @@ +from pathlib import Path + +import msgpack # type: ignore + +import ufoLib2.objects +import ufoLib2.serde.msgpack + + +def test_dumps_loads(ufo_UbuTestData: ufoLib2.objects.Font) -> None: + font = ufo_UbuTestData + data = font.msgpack_dumps() # type: ignore + + assert data[:10] == b"\x85\xa6layers\x92\x82" + + font2 = ufoLib2.objects.Font.msgpack_loads(data) # type: ignore + + assert font == font2 + + +def test_dump_load(tmp_path: Path, ufo_UbuTestData: ufoLib2.objects.Font) -> None: + + font = ufo_UbuTestData + with open(tmp_path / "test.msgpack", "wb") as f: + font.msgpack_dump(f) # type: ignore + + with open(tmp_path / "test.msgpack", "rb") as f: + font2 = ufoLib2.objects.Font.msgpack_load(f) # type: ignore + + assert font == font2 + + +def test_allow_bytes(ufo_UbuTestData: ufoLib2.objects.Font) -> None: + font = ufo_UbuTestData + + # DataSet values are binary data (bytes) + assert all(isinstance(v, bytes) for v in font.data.values()) + + # bytes *are* allowed in MessagePack (unlike JSON), so its converter should + # keep them as such (not translate them to Base64 str) upon serializig + b = font.data.msgpack_dumps() # type: ignore + + # check that (even before structuring the DataSet object) the msgpack raw data + # contains in fact bytes, not str + raw_data = msgpack.unpackb(b) + assert isinstance(raw_data, dict) + assert all(isinstance(v, bytes) for v in raw_data.values()) + + # of course, also after structuring, the DataSet should contain bytes + data = font.data.msgpack_loads(b) # type: ignore + assert isinstance(data, ufoLib2.objects.DataSet) + assert all(isinstance(v, bytes) for v in data.values()) diff --git a/tests/serde/test_pickle.py b/tests/serde/test_pickle.py new file mode 100644 index 00000000..29252045 --- /dev/null +++ b/tests/serde/test_pickle.py @@ -0,0 +1,59 @@ +import pickle +from pathlib import Path +from typing import Any + +import pytest + +import ufoLib2.objects +import ufoLib2.serde.pickle + + +@pytest.fixture +def font(request: Any, datadir: Path) -> ufoLib2.Font: + lazy = request.param + if lazy is not None: + return ufoLib2.Font.open(datadir / "UbuTestData.ufo", lazy=lazy) + else: + return ufoLib2.Font.open(datadir / "UbuTestData.ufo") + + +@pytest.mark.parametrize( + "font", [None, False, True], ids=["lazy-unset", "non-lazy", "lazy"], indirect=True +) +def test_dumps_loads(font: ufoLib2.objects.Font) -> None: + orig_lazy = font._lazy + + data = font.pickle_dumps() # type: ignore + + assert isinstance(data, bytes) and len(data) > 0 + + # picklying unlazifies + if orig_lazy: + assert font._lazy is False + else: + assert font._lazy is orig_lazy + + font2 = ufoLib2.objects.Font.pickle_loads(data) # type: ignore + + assert font == font2 + # unpickling doesn't initialize the lazy flag, which resets to default + assert font2._lazy is None + + # Font.pickle_loads(s) is just syntactic sugar for pickle.loads(s) anyway + font3 = pickle.loads(data) + assert font == font3 + # same for font.pickle_dumps() => pickle.dumps(font) + assert data == pickle.dumps(font3) + + +@pytest.mark.parametrize( + "font", [None, False, True], ids=["lazy-unset", "non-lazy", "lazy"], indirect=True +) +def test_dump_load(tmp_path: Path, font: ufoLib2.objects.Font) -> None: + with open(tmp_path / "test.pickle", "wb") as f: + font.pickle_dump(f) # type: ignore + + with open(tmp_path / "test.pickle", "rb") as f: + font2 = ufoLib2.objects.Font.pickle_load(f) # type: ignore + + assert font == font2 diff --git a/tests/serde/test_serde.py b/tests/serde/test_serde.py new file mode 100644 index 00000000..21ee9c3f --- /dev/null +++ b/tests/serde/test_serde.py @@ -0,0 +1,29 @@ +import importlib +import sys +from typing import Any + +import pytest +from attrs import define + +from ufoLib2.serde import serde + + +def test_raise_import_error(monkeypatch: Any) -> None: + # pretent we can't import the module (e.g. msgpack not installed) + monkeypatch.setitem(sys.modules, "ufoLib2.serde.msgpack", None) + + with pytest.raises(ImportError, match="ufoLib2.serde.msgpack"): + importlib.import_module("ufoLib2.serde.msgpack") + + @serde + @define + class Foo: + a: int + b: str = "bar" + + foo = Foo(1) + + with pytest.raises(ImportError, match="ufoLib2.serde.msgpack"): + # since the method is only added dynamicall at runtime, mypy complains that + # "Foo" has no attribute "msgpack_dumps" -- so I shut it up + foo.msgpack_dumps() # type: ignore From 34429694ffeaa1dea1cfcf0f3e61e33955931ed6 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 19:13:21 +0200 Subject: [PATCH 07/22] skip test if extras not installed; force mypy to not typecheck orjson --- tests/serde/test_json.py | 6 +++++- tests/serde/test_msgpack.py | 7 ++++++- tox.ini | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/serde/test_json.py b/tests/serde/test_json.py index bcda8847..88d4c502 100644 --- a/tests/serde/test_json.py +++ b/tests/serde/test_json.py @@ -7,7 +7,11 @@ import pytest import ufoLib2.objects -import ufoLib2.serde.json + +# isort: off +pytest.importorskip("cattr") + +import ufoLib2.serde.json # noqa: E402 @pytest.mark.parametrize("have_orjson", [False, True], ids=["no-orjson", "with-orjson"]) diff --git a/tests/serde/test_msgpack.py b/tests/serde/test_msgpack.py index eeebf045..55ef4117 100644 --- a/tests/serde/test_msgpack.py +++ b/tests/serde/test_msgpack.py @@ -1,9 +1,14 @@ from pathlib import Path import msgpack # type: ignore +import pytest import ufoLib2.objects -import ufoLib2.serde.msgpack + +# isort: off +pytest.importorskip("cattr") + +import ufoLib2.serde.msgpack # noqa: E402 def test_dumps_loads(ufo_UbuTestData: ufoLib2.objects.Font) -> None: diff --git a/tox.ini b/tox.ini index e55a83b3..8d418c3e 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,8 @@ deps = commands = black --check --diff . isort --skip-gitignore --check-only --diff src tests + # typing stubs for orjson exist but I don't want mypy to use them! + pip uninstall -y orjson mypy --strict src tests flake8 From 0dfc108b27f2f0468f4a121c5462bbbc15703f7b Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 19:37:50 +0200 Subject: [PATCH 08/22] info: forgot to add @serde decorator on Info class! --- src/ufoLib2/objects/info/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ufoLib2/objects/info/__init__.py b/src/ufoLib2/objects/info/__init__.py index ac75d0f9..bcae6898 100644 --- a/src/ufoLib2/objects/info/__init__.py +++ b/src/ufoLib2/objects/info/__init__.py @@ -10,6 +10,7 @@ from ufoLib2.objects.guideline import Guideline from ufoLib2.objects.misc import AttrDictMixin +from ufoLib2.serde import serde from .woff import ( WoffMetadataCopyright, @@ -170,6 +171,7 @@ def _dict_list_setter_property(cls: type[Tc], name: str | None = None) -> Any: ) +@serde @define class Info: """A data class representing the contents of fontinfo.plist. From fb329946510fbe166a51b938fa27f3d265458ee6 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 19:46:12 +0200 Subject: [PATCH 09/22] test de/serializing ALL ufoLib2.objects and in fact I had forgotten to decorate one of them (Info) ;) --- tests/serde/test_serde.py | 44 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/serde/test_serde.py b/tests/serde/test_serde.py index 21ee9c3f..4588db6f 100644 --- a/tests/serde/test_serde.py +++ b/tests/serde/test_serde.py @@ -1,11 +1,12 @@ import importlib import sys -from typing import Any +from typing import Any, Dict, List import pytest from attrs import define -from ufoLib2.serde import serde +import ufoLib2.objects +from ufoLib2.serde import _SERDE_FORMATS_, serde def test_raise_import_error(monkeypatch: Any) -> None: @@ -27,3 +28,42 @@ class Foo: # since the method is only added dynamicall at runtime, mypy complains that # "Foo" has no attribute "msgpack_dumps" -- so I shut it up foo.msgpack_dumps() # type: ignore + + +BASIC_EMPTY_OBJECTS: List[Dict[str, Any]] = [ + {"class_name": "Anchor", "args": (0, 0)}, + {"class_name": "Component", "args": ("a",)}, + {"class_name": "Contour", "args": ()}, + {"class_name": "DataSet", "args": ()}, + {"class_name": "Features", "args": ()}, + {"class_name": "Font", "args": ()}, + {"class_name": "Glyph", "args": ()}, + {"class_name": "Guideline", "args": (1,)}, + {"class_name": "Image", "args": ()}, + {"class_name": "ImageSet", "args": ()}, + {"class_name": "Info", "args": ()}, + {"class_name": "Kerning", "args": ()}, + {"class_name": "Layer", "args": ()}, + { + "class_name": "LayerSet", + "args": ({"public.default": ufoLib2.objects.Layer()},), + }, + {"class_name": "Lib", "args": ()}, + {"class_name": "Point", "args": (2, 3)}, +] +assert {d["class_name"] for d in BASIC_EMPTY_OBJECTS} == set(ufoLib2.objects.__all__) + + +@pytest.mark.parametrize("fmt", _SERDE_FORMATS_) +@pytest.mark.parametrize( + "object_info", + BASIC_EMPTY_OBJECTS, + ids=lambda x: x["class_name"], # type: ignore +) +def test_serde_all_objects(fmt: str, object_info: Dict[str, Any]) -> None: + klass = getattr(ufoLib2.objects, object_info["class_name"]) + loads = getattr(klass, f"{fmt}_loads") + obj = klass(*object_info["args"]) + dumps = getattr(obj, f"{fmt}_dumps") + obj2 = loads(dumps()) + assert obj == obj2 From 30f639fdc01030caf71df021f21da4836677ed6c Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Wed, 27 Jul 2022 19:51:41 +0200 Subject: [PATCH 10/22] skip format-specific tests when cattrs can't be imported --- tests/serde/test_serde.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/serde/test_serde.py b/tests/serde/test_serde.py index 4588db6f..d34442cb 100644 --- a/tests/serde/test_serde.py +++ b/tests/serde/test_serde.py @@ -61,6 +61,10 @@ class Foo: ids=lambda x: x["class_name"], # type: ignore ) def test_serde_all_objects(fmt: str, object_info: Dict[str, Any]) -> None: + if fmt in ("json", "msgpack"): + # skip these format tests if cattrs is not installed + pytest.importorskip("cattr") + klass = getattr(ufoLib2.objects, object_info["class_name"]) loads = getattr(klass, f"{fmt}_loads") obj = klass(*object_info["args"]) From 0ac647b4230d6dfb4b2c9b66f1128fb94c58439e Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 28 Jul 2022 10:48:09 +0200 Subject: [PATCH 11/22] remove ufoLib2.serde.pickle, doesn't add anything Doing font.pickle_dumps() is exactly the same as doing pickle.dumps(font), same for Font.pickle_loads(s) vs pickle.loads(s), we don't use cattrs for pickling. So it's better to simply remove the extra code, pickling still works out of the box (even with lazy objects now). --- src/ufoLib2/serde/__init__.py | 43 +++++++++++++++++++------ src/ufoLib2/serde/pickle.py | 16 ---------- tests/objects/test_font.py | 34 ++++++++++++++++++++ tests/serde/test_pickle.py | 59 ----------------------------------- 4 files changed, 67 insertions(+), 85 deletions(-) delete mode 100644 src/ufoLib2/serde/pickle.py delete mode 100644 tests/serde/test_pickle.py diff --git a/src/ufoLib2/serde/__init__.py b/src/ufoLib2/serde/__init__.py index 5356f39f..6a9b21d1 100644 --- a/src/ufoLib2/serde/__init__.py +++ b/src/ufoLib2/serde/__init__.py @@ -6,7 +6,7 @@ from ufoLib2.typing import PathLike, T -_SERDE_FORMATS_ = ("json", "msgpack", "pickle") +_SERDE_FORMATS_ = ("json", "msgpack") def _loads( @@ -53,19 +53,42 @@ def _dump( def serde(cls: Type[T]) -> Type[T]: """Decorator to add serialization support to a ufoLib2 class. - Currently JSON, MessagePack (msgpack) and Pickle are the supported formats, - but other formats may be added in the future. + This adds f"{format}_loads" / f"{format}_dumps" (from/to bytes) methods, and + f"{format}_load" / f"{format}_dump" (for file or path) methods to all ufoLib2 + objects, not just Font. - Pickle works out of the box, whereas the others require additional extras - to be installed: e.g. ufoLib2[json,msgpack]. If required, this will install - the `cattrs` library for structuring/unstructuring custom objects from/to - serializable data structures (also available with ufoLib2[converters] extra). + Currently the supported formats are JSON and MessagePack (msgpack), but other + formats may be added in the future. - If any of the optional dependencies fails to be imported, this decorator will - raise an ImportError when any of the related methods are called. + E.g.:: + + from ufoLib2 import Font + + font = Font.open("MyFont.ufo") + font.json_dump("MyFont.json") + font2 = Font.json_load("MyFont.json") + font3 = Font.json_loads(font2.json_dumps()) + + font3.msgpack_dump("MyFont.msgpack") + font4 = Font.msgpack_load("MyFont.msgpack") + # etc. + + Note this requires additional extras to be installed: e.g. ufoLib2[json,msgpack]. + In additions to the respective serialization library, these installs the `cattrs` + library for structuring/unstructuring custom objects from/to serializable data + structures (also available separately as ufoLib2[converters] extra). + + If any of the optional dependencies fails to be imported, the methods will raise + an ImportError when called. If the faster `orjson` library is present, it will be used in place of the - built-in `json` library. + built-in `json` library on CPython. On PyPy, the `orjson` library is not available, + so the built-in `json` library will be used (though it's pretty fast anyway). + + If you want a serialization format that works out of the box with all ufoLib2 + objects (but it's mostly limited to Python) you can use the built-in pickle module, + which doesn't require to use the `cattrs` converters. + """ supported_formats = [] diff --git a/src/ufoLib2/serde/pickle.py b/src/ufoLib2/serde/pickle.py deleted file mode 100644 index 40ee1bab..00000000 --- a/src/ufoLib2/serde/pickle.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -import pickle -from typing import Any, Type - -from ufoLib2.typing import T - - -def dumps(obj: Any, **kwargs: Any) -> bytes: - return pickle.dumps(obj, **kwargs) - - -def loads(s: bytes, object_class: Type[T], **kwargs: Any) -> T: - obj = pickle.loads(s, **kwargs) - assert isinstance(obj, object_class) - return obj diff --git a/tests/objects/test_font.py b/tests/objects/test_font.py index 24c1b6d8..6a7d49a1 100644 --- a/tests/objects/test_font.py +++ b/tests/objects/test_font.py @@ -1,6 +1,10 @@ from __future__ import annotations +import pickle from pathlib import Path +from typing import Optional + +import pytest from ufoLib2.objects import Font, Glyph, Guideline @@ -101,3 +105,33 @@ def test_data_images_init() -> None: assert font.data["bbb/c"] == b"456" assert font.images["a.png"] == b"\x89PNG\r\n\x1a\n" assert font.images["b.png"] == b"\x89PNG\r\n\x1a\n" + + +@pytest.mark.parametrize( + "lazy", [None, False, True], ids=["lazy-unset", "non-lazy", "lazy"] +) +def test_pickle_lazy_font(datadir: Path, lazy: Optional[bool]) -> None: + if lazy is not None: + font = Font.open(datadir / "UbuTestData.ufo", lazy=lazy) + else: + font = Font() + + assert lazy is font._lazy + + data = pickle.dumps(font) + + assert isinstance(data, bytes) and len(data) > 0 + + # picklying unlazifies + if lazy: + assert font._lazy is False + else: + assert font._lazy is lazy + + font2 = pickle.loads(data) + + assert isinstance(font2, Font) + assert font == font2 + # unpickling doesn't initialize the lazy flag or a reader, which reset to default + assert font2._lazy is None + assert font2._reader is None diff --git a/tests/serde/test_pickle.py b/tests/serde/test_pickle.py deleted file mode 100644 index 29252045..00000000 --- a/tests/serde/test_pickle.py +++ /dev/null @@ -1,59 +0,0 @@ -import pickle -from pathlib import Path -from typing import Any - -import pytest - -import ufoLib2.objects -import ufoLib2.serde.pickle - - -@pytest.fixture -def font(request: Any, datadir: Path) -> ufoLib2.Font: - lazy = request.param - if lazy is not None: - return ufoLib2.Font.open(datadir / "UbuTestData.ufo", lazy=lazy) - else: - return ufoLib2.Font.open(datadir / "UbuTestData.ufo") - - -@pytest.mark.parametrize( - "font", [None, False, True], ids=["lazy-unset", "non-lazy", "lazy"], indirect=True -) -def test_dumps_loads(font: ufoLib2.objects.Font) -> None: - orig_lazy = font._lazy - - data = font.pickle_dumps() # type: ignore - - assert isinstance(data, bytes) and len(data) > 0 - - # picklying unlazifies - if orig_lazy: - assert font._lazy is False - else: - assert font._lazy is orig_lazy - - font2 = ufoLib2.objects.Font.pickle_loads(data) # type: ignore - - assert font == font2 - # unpickling doesn't initialize the lazy flag, which resets to default - assert font2._lazy is None - - # Font.pickle_loads(s) is just syntactic sugar for pickle.loads(s) anyway - font3 = pickle.loads(data) - assert font == font3 - # same for font.pickle_dumps() => pickle.dumps(font) - assert data == pickle.dumps(font3) - - -@pytest.mark.parametrize( - "font", [None, False, True], ids=["lazy-unset", "non-lazy", "lazy"], indirect=True -) -def test_dump_load(tmp_path: Path, font: ufoLib2.objects.Font) -> None: - with open(tmp_path / "test.pickle", "wb") as f: - font.pickle_dump(f) # type: ignore - - with open(tmp_path / "test.pickle", "rb") as f: - font2 = ufoLib2.objects.Font.pickle_load(f) # type: ignore - - assert font == font2 From 6378e56c73626ea7799b31a6e5bc9180a9cb0bcc Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 28 Jul 2022 11:08:18 +0200 Subject: [PATCH 12/22] also test {json,msgpack}_load/dump with paths, in addition to files --- tests/serde/test_json.py | 18 ++++++++++++++++++ tests/serde/test_msgpack.py | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/serde/test_json.py b/tests/serde/test_json.py index 88d4c502..dd47bef8 100644 --- a/tests/serde/test_json.py +++ b/tests/serde/test_json.py @@ -63,6 +63,24 @@ def test_dump_load( assert font == font2 + with open(tmp_path / "test.json", "wb") as f: + font.json_dump(f, indent=indent, sort_keys=sort_keys) # type: ignore + + # load/dump work with paths too, not just file objects + font3 = ufoLib2.objects.Font.json_load(tmp_path / "test.json") # type: ignore + + assert font == font3 + + font.json_dump( # type: ignore + tmp_path / "test2.json", + indent=indent, + sort_keys=sort_keys, + ) + + assert (tmp_path / "test.json").read_bytes() == ( + tmp_path / "test2.json" + ).read_bytes() + @pytest.mark.parametrize("indent", [1, 3], ids=["indent-1", "indent-3"]) def test_indent_not_2_orjson(indent: int) -> None: diff --git a/tests/serde/test_msgpack.py b/tests/serde/test_msgpack.py index 55ef4117..6a697015 100644 --- a/tests/serde/test_msgpack.py +++ b/tests/serde/test_msgpack.py @@ -33,6 +33,17 @@ def test_dump_load(tmp_path: Path, ufo_UbuTestData: ufoLib2.objects.Font) -> Non assert font == font2 + # laod/dump work with paths too, not just file objects + font3 = ufoLib2.objects.Font.msgpack_load(tmp_path / "test.msgpack") # type: ignore + + assert font == font3 + + font.msgpack_dump(tmp_path / "test2.msgpack") # type: ignore + + assert (tmp_path / "test.msgpack").read_bytes() == ( + tmp_path / "test2.msgpack" + ).read_bytes() + def test_allow_bytes(ufo_UbuTestData: ufoLib2.objects.Font) -> None: font = ufo_UbuTestData From 5a8d9bc064b1b7e5c8166ab89234f90c42e327f4 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 3 Nov 2022 18:10:36 +0000 Subject: [PATCH 13/22] change custom {get,set}state to use dict, like upstream attrs does Follows https://github.com/python-attrs/attrs/pull/1009 --- src/ufoLib2/objects/misc.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index 2248d46e..5f6b4003 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -84,13 +84,13 @@ def _deepcopy_unlazify_attrs(self: Any, memo: Any) -> Any: ) -def _getstate_unlazify_attrs(self: Any) -> Tuple[Any, ...]: +def _getstate_unlazify_attrs(self: Any) -> Dict[str, Any]: if self._lazy: self.unlazify() - return tuple( - getattr(self, a.name) if a.init else a.default + return { + a.name: getattr(self, a.name) if a.init else a.default for a in attr.fields(self.__class__) - ) + } _obj_setattr = object.__setattr__ @@ -99,10 +99,11 @@ def _getstate_unlazify_attrs(self: Any) -> Tuple[Any, ...]: # Since we override __getstate__, we must also override __setstate__. # Below is adapted from `attrs._make._ClassBuilder._make_getstate_setstate` method: # https://github.com/python-attrs/attrs/blob/36ed0204/src/attr/_make.py#L931-L937 -def _setstate_attrs(self: Any, state: Tuple[Any, ...]) -> None: +def _setstate_attrs(self: Any, state: Dict[str, Any]) -> None: _bound_setattr = _obj_setattr.__get__(self, attr.Attribute) # type: ignore - for a, v in zip(attr.fields(self.__class__), state): - _bound_setattr(a.name, v) + for a in attr.fields(self.__class__): + if a.name in state: + _bound_setattr(a.name, state[a.name]) def _object_lib(parent_lib: dict[str, Any], obj: HasIdentifier) -> dict[str, Any]: From a8223ed9c4f0885c614957b675b0056c4d968d4b Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 3 Nov 2022 18:30:31 +0000 Subject: [PATCH 14/22] require latest attrs/cattrs; rename GenConverter => Converter in latest cattrs, GenConverter _is_ the normal Converter anyway --- requirements.txt | 6 +++--- setup.cfg | 6 +++--- src/ufoLib2/converters.py | 10 +++++----- src/ufoLib2/objects/features.py | 6 +++--- src/ufoLib2/objects/kerning.py | 6 +++--- src/ufoLib2/objects/layer.py | 6 +++--- src/ufoLib2/objects/layerSet.py | 6 +++--- src/ufoLib2/objects/lib.py | 10 +++++----- src/ufoLib2/objects/misc.py | 6 +++--- tests/test_converters.py | 8 ++++---- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/requirements.txt b/requirements.txt index 96d5fb7e..15865dcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --extra=converters --extra=json --extra=lxml --extra=msgpack setup.cfg # appdirs==1.4.4 # via fs -attrs==21.4.0 +attrs==22.1.0 # via # cattrs # ufoLib2 (setup.cfg) -cattrs==1.10.0 +cattrs==22.2.0 # via ufoLib2 (setup.cfg) fonttools[ufo]==4.29.1 # via ufoLib2 (setup.cfg) diff --git a/setup.cfg b/setup.cfg index 99a7c881..514f3216 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ package_dir = =src packages = find: python_requires = >=3.7 install_requires = - attrs >= 20.1.0 + attrs >= 22.1.0 fonttools[ufo] >= 4.0.0 typing_extensions ; python_version < "3.8" @@ -34,10 +34,10 @@ install_requires = lxml = lxml converters = cattrs >= 1.10.0 json = - cattrs >= 1.1.0 + cattrs >= 22.2.0 orjson ; platform_python_implementation != 'PyPy' msgpack = - cattrs >= 1.1.0 + cattrs >= 22.2.0 msgpack [options.packages.find] diff --git a/src/ufoLib2/converters.py b/src/ufoLib2/converters.py index f6e84495..6472bc04 100644 --- a/src/ufoLib2/converters.py +++ b/src/ufoLib2/converters.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Tuple, Type, cast from attr import fields, has, resolve_types -from cattr import GenConverter +from cattr import Converter from cattr.gen import ( AttributeOverride, make_dict_structure_fn, @@ -49,7 +49,7 @@ def is_ufoLib2_class_with_custom_structure(cls: Type[Any]) -> bool: return is_ufoLib2_class(cls) and hasattr(cls, "_structure") -def register_hooks(conv: GenConverter, allow_bytes: bool = True) -> None: +def register_hooks(conv: Converter, allow_bytes: bool = True) -> None: def attrs_hook_factory( cls: Type[Any], gen_fn: Callable[..., Callable[[Any], Any]], structuring: bool ) -> Callable[[Any], Any]: @@ -71,7 +71,7 @@ def attrs_hook_factory( # classes that don't have a custom hook registered) check for any # type_overrides (Dict[Type, AttributeOverride]); they allow a custom # converter to omit specific attributes of given type e.g.: - # >>> conv = GenConverter(type_overrides={Image: override(omit=True)}) + # >>> conv = Converter(type_overrides={Image: override(omit=True)}) attrib_override = conv.type_overrides[a.type] else: # by default, we omit all Optional attributes (i.e. with None default), @@ -134,7 +134,7 @@ def structure_bytes(v: str, _: Any) -> bytes: conv.register_structure_hook(bytes, structure_bytes) -default_converter = GenConverter( +default_converter = Converter( omit_if_default=True, forbid_extra_keys=True, prefer_attrib_converters=False, @@ -145,7 +145,7 @@ def structure_bytes(v: str, _: Any) -> bytes: unstructure = default_converter.unstructure # same as default_converter but allows bytes -binary_converter = GenConverter( +binary_converter = Converter( omit_if_default=True, forbid_extra_keys=True, prefer_attrib_converters=False, diff --git a/src/ufoLib2/objects/features.py b/src/ufoLib2/objects/features.py index 265eebbf..133dba4b 100644 --- a/src/ufoLib2/objects/features.py +++ b/src/ufoLib2/objects/features.py @@ -8,7 +8,7 @@ from ufoLib2.serde import serde if TYPE_CHECKING: - from cattr import GenConverter + from cattr import Converter RE_NEWLINES = re.compile(r"\r\n|\r") @@ -36,11 +36,11 @@ def normalize_newlines(self) -> Features: self.text = RE_NEWLINES.sub("\n", self.text) return self - def _unstructure(self, converter: GenConverter) -> str: + def _unstructure(self, converter: Converter) -> str: del converter # unused return self.text @staticmethod - def _structure(data: str, cls: Type[Features], converter: GenConverter) -> Features: + def _structure(data: str, cls: Type[Features], converter: Converter) -> Features: del converter # unused return cls(data) diff --git a/src/ufoLib2/objects/kerning.py b/src/ufoLib2/objects/kerning.py index 1e14d43c..e183dd54 100644 --- a/src/ufoLib2/objects/kerning.py +++ b/src/ufoLib2/objects/kerning.py @@ -7,7 +7,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import GenConverter + from cattr import Converter KerningPair = Tuple[str, str] @@ -28,7 +28,7 @@ def from_nested_dicts(self, kerning: Mapping[str, Mapping[str, float]]) -> Kerni for right in kerning[left] ) - def _unstructure(self, converter: GenConverter) -> dict[str, dict[str, float]]: + def _unstructure(self, converter: Converter) -> dict[str, dict[str, float]]: del converter # unused return self.as_nested_dicts() @@ -36,7 +36,7 @@ def _unstructure(self, converter: GenConverter) -> dict[str, dict[str, float]]: def _structure( data: Mapping[str, Mapping[str, float]], cls: Type[Kerning], - converter: GenConverter, + converter: Converter, ) -> Kerning: del converter # unused return cls.from_nested_dicts(data) diff --git a/src/ufoLib2/objects/layer.py b/src/ufoLib2/objects/layer.py index 6d882dc2..c873c2eb 100644 --- a/src/ufoLib2/objects/layer.py +++ b/src/ufoLib2/objects/layer.py @@ -30,7 +30,7 @@ from ufoLib2.typing import T if TYPE_CHECKING: - from cattr import GenConverter + from cattr import Converter _GLYPH_NOT_LOADED = Glyph(name="___UFOLIB2_LAZY_GLYPH___") @@ -382,7 +382,7 @@ def write(self, glyphSet: GlyphSet, saveAs: bool = True) -> None: # all glyphs are loaded by now, no need to keep ref to glyphSet self._glyphSet = None - def _unstructure(self, converter: GenConverter) -> dict[str, Any]: + def _unstructure(self, converter: Converter) -> dict[str, Any]: # omit glyph name attribute, already used as key glyphs: dict[str, dict[str, Any]] = {} for glyph_name in self._glyphs: @@ -408,7 +408,7 @@ def _unstructure(self, converter: GenConverter) -> dict[str, Any]: @staticmethod def _structure( - data: dict[str, Any], cls: Type[Layer], converter: GenConverter + data: dict[str, Any], cls: Type[Layer], converter: Converter ) -> Layer: return cls( name=data.get("name", DEFAULT_LAYER_NAME), diff --git a/src/ufoLib2/objects/layerSet.py b/src/ufoLib2/objects/layerSet.py index 654846c3..e033cbdd 100644 --- a/src/ufoLib2/objects/layerSet.py +++ b/src/ufoLib2/objects/layerSet.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import GenConverter + from cattr import Converter _LAYER_NOT_LOADED = Layer(name="___UFOLIB2_LAZY_LAYER___") @@ -387,11 +387,11 @@ def write(self, writer: UFOWriter, saveAs: bool | None = None) -> None: layer.write(glyphSet, saveAs=saveAs) writer.writeLayerContents(self.layerOrder) - def _unstructure(self, converter: GenConverter) -> list[dict[str, Any]]: + def _unstructure(self, converter: Converter) -> list[dict[str, Any]]: return [converter.unstructure(layer) for layer in self] @staticmethod def _structure( - data: list[dict[str, Any]], cls: Type[LayerSet], converter: GenConverter + data: list[dict[str, Any]], cls: Type[LayerSet], converter: Converter ) -> LayerSet: return cls.from_iterable(converter.structure(layer, Layer) for layer in data) diff --git a/src/ufoLib2/objects/lib.py b/src/ufoLib2/objects/lib.py index 769d8d31..23b54d60 100644 --- a/src/ufoLib2/objects/lib.py +++ b/src/ufoLib2/objects/lib.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import GenConverter + from cattr import Converter # unfortunately mypy is not smart enough to support recursive types like plist... # PlistEncodable = Union[ @@ -45,7 +45,7 @@ def is_data_dict(value: Any) -> bool: ) -def _unstructure_data(value: Any, converter: GenConverter) -> Any: +def _unstructure_data(value: Any, converter: Converter) -> Any: if isinstance(value, bytes): return {"type": DATA_LIB_KEY, "data": converter.unstructure(value)} elif isinstance(value, (list, tuple)): @@ -56,7 +56,7 @@ def _unstructure_data(value: Any, converter: GenConverter) -> Any: def _structure_data_inplace( - key: Union[int, str], value: Any, container: Any, converter: GenConverter + key: Union[int, str], value: Any, container: Any, converter: Converter ) -> None: if isinstance(value, list): for i, v in enumerate(value): @@ -70,7 +70,7 @@ def _structure_data_inplace( @serde class Lib(Dict[str, Any]): - def _unstructure(self, converter: GenConverter) -> dict[str, Any]: + def _unstructure(self, converter: Converter) -> dict[str, Any]: # avoid encoding if converter supports bytes natively test = converter.unstructure(b"\0") if isinstance(test, bytes): @@ -85,7 +85,7 @@ def _unstructure(self, converter: GenConverter) -> dict[str, Any]: def _structure( data: Mapping[str, Any], cls: Type[Lib], - converter: GenConverter, + converter: Converter, ) -> Lib: self = cls(data) for k, v in self.items(): diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index 5f6b4003..eb092cc7 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -32,7 +32,7 @@ from ufoLib2.typing import Drawable, GlyphSet, HasIdentifier if TYPE_CHECKING: - from cattr import GenConverter + from cattr import Converter class BoundingBox(NamedTuple): @@ -314,7 +314,7 @@ def fileNames(self) -> list[str]: """Returns a list of filenames in the data store.""" return list(self._data.keys()) - def _unstructure(self, converter: GenConverter) -> dict[str, str]: + def _unstructure(self, converter: Converter) -> dict[str, str]: # avoid encoding if converter supports bytes natively test = converter.unstructure(b"\0") if isinstance(test, bytes): @@ -335,7 +335,7 @@ def _unstructure(self, converter: GenConverter) -> dict[str, str]: def _structure( data: Mapping[str, Any], cls: Type[DataStore], - converter: GenConverter, + converter: Converter, ) -> DataStore: self = cls() for k, v in data.items(): diff --git a/tests/test_converters.py b/tests/test_converters.py index 4e1bfc4e..0e55f000 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -517,7 +517,7 @@ def test_unstructure_lazy_font(ufo_UbuTestData: Font) -> None: @pytest.mark.parametrize("forbid_extra_keys", [True, False]) def test_structure_forbid_extra_keys(forbid_extra_keys: bool) -> None: - conv = cattr.GenConverter(forbid_extra_keys=forbid_extra_keys) + conv = cattr.Converter(forbid_extra_keys=forbid_extra_keys) register_hooks(conv) data = {"name": "a", "foo": "bar"} if forbid_extra_keys: @@ -599,7 +599,7 @@ def test_structure_forbid_extra_keys(forbid_extra_keys: bool) -> None: ], ) def test_omit_if_default(obj: Any, expected: Any, omit_if_default: bool) -> None: - conv = cattr.GenConverter(omit_if_default=omit_if_default) + conv = cattr.Converter(omit_if_default=omit_if_default) register_hooks(conv) assert conv.unstructure(obj) == expected @@ -653,7 +653,7 @@ def test_omit_if_default(obj: Any, expected: Any, omit_if_default: bool) -> None ], ) def test_allow_bytes(obj: Any, expected: Any, allow_bytes: bool) -> None: - conv = cattr.GenConverter() + conv = cattr.Converter() register_hooks(conv, allow_bytes=allow_bytes) assert conv.unstructure(obj) == expected @@ -661,7 +661,7 @@ def test_allow_bytes(obj: Any, expected: Any, allow_bytes: bool) -> None: def test_custom_type_overrides() -> None: - conv = cattr.GenConverter(type_overrides={Image: cattr.override(omit=True)}) + conv = cattr.Converter(type_overrides={Image: cattr.override(omit=True)}) register_hooks(conv) # check that Glyph.image attribute (of type Image) is omitted From b421da32b924c04a047af1fdb41236a0bc0664a5 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 3 Nov 2022 18:57:13 +0000 Subject: [PATCH 15/22] import attrs/cattrs from the new namespaces with the 's' at the end --- src/ufoLib2/converters.py | 6 +++--- src/ufoLib2/objects/anchor.py | 2 +- src/ufoLib2/objects/component.py | 2 +- src/ufoLib2/objects/contour.py | 2 +- src/ufoLib2/objects/features.py | 4 ++-- src/ufoLib2/objects/font.py | 2 +- src/ufoLib2/objects/glyph.py | 2 +- src/ufoLib2/objects/guideline.py | 2 +- src/ufoLib2/objects/image.py | 2 +- src/ufoLib2/objects/info/__init__.py | 6 +++--- src/ufoLib2/objects/info/woff.py | 2 +- src/ufoLib2/objects/kerning.py | 2 +- src/ufoLib2/objects/layer.py | 4 ++-- src/ufoLib2/objects/layerSet.py | 4 ++-- src/ufoLib2/objects/lib.py | 2 +- src/ufoLib2/objects/misc.py | 18 +++++++++--------- src/ufoLib2/objects/point.py | 2 +- tests/serde/test_json.py | 2 +- tests/serde/test_msgpack.py | 2 +- tests/serde/test_serde.py | 2 +- tests/test_converters.py | 10 +++++----- 21 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/ufoLib2/converters.py b/src/ufoLib2/converters.py index 6472bc04..d6ced211 100644 --- a/src/ufoLib2/converters.py +++ b/src/ufoLib2/converters.py @@ -4,9 +4,9 @@ from functools import partial from typing import Any, Callable, Tuple, Type, cast -from attr import fields, has, resolve_types -from cattr import Converter -from cattr.gen import ( +from attrs import fields, has, resolve_types +from cattrs import Converter +from cattrs.gen import ( AttributeOverride, make_dict_structure_fn, make_dict_unstructure_fn, diff --git a/src/ufoLib2/objects/anchor.py b/src/ufoLib2/objects/anchor.py index b31d67d0..c2d9d3b3 100644 --- a/src/ufoLib2/objects/anchor.py +++ b/src/ufoLib2/objects/anchor.py @@ -2,7 +2,7 @@ from typing import Optional -from attr import define +from attrs import define from ufoLib2.objects.misc import AttrDictMixin from ufoLib2.serde import serde diff --git a/src/ufoLib2/objects/component.py b/src/ufoLib2/objects/component.py index b67945c7..f5c60208 100644 --- a/src/ufoLib2/objects/component.py +++ b/src/ufoLib2/objects/component.py @@ -3,7 +3,7 @@ import warnings from typing import Optional -from attr import define, field +from attrs import define, field from fontTools.misc.transform import Identity, Transform from fontTools.pens.basePen import AbstractPen from fontTools.pens.pointPen import AbstractPointPen, PointToSegmentPen diff --git a/src/ufoLib2/objects/contour.py b/src/ufoLib2/objects/contour.py index a53e971e..6a63e1f0 100644 --- a/src/ufoLib2/objects/contour.py +++ b/src/ufoLib2/objects/contour.py @@ -4,7 +4,7 @@ from collections.abc import MutableSequence from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, overload -from attr import define, field +from attrs import define, field from fontTools.pens.basePen import AbstractPen from fontTools.pens.pointPen import AbstractPointPen, PointToSegmentPen diff --git a/src/ufoLib2/objects/features.py b/src/ufoLib2/objects/features.py index 133dba4b..220e012d 100644 --- a/src/ufoLib2/objects/features.py +++ b/src/ufoLib2/objects/features.py @@ -3,12 +3,12 @@ import re from typing import TYPE_CHECKING, Type -from attr import define +from attrs import define from ufoLib2.serde import serde if TYPE_CHECKING: - from cattr import Converter + from cattrs import Converter RE_NEWLINES = re.compile(r"\r\n|\r") diff --git a/src/ufoLib2/objects/font.py b/src/ufoLib2/objects/font.py index 03d530de..0c227f49 100644 --- a/src/ufoLib2/objects/font.py +++ b/src/ufoLib2/objects/font.py @@ -18,7 +18,7 @@ import fs.base import fs.tempfs -from attr import define, field +from attrs import define, field from fontTools.ufoLib import UFOFileStructure, UFOReader, UFOWriter from ufoLib2.constants import DEFAULT_LAYER_NAME diff --git a/src/ufoLib2/objects/glyph.py b/src/ufoLib2/objects/glyph.py index d87ea0a4..b0b10d10 100644 --- a/src/ufoLib2/objects/glyph.py +++ b/src/ufoLib2/objects/glyph.py @@ -3,7 +3,7 @@ from copy import deepcopy from typing import Any, Iterator, List, Mapping, Optional, cast -from attr import define, field +from attrs import define, field from fontTools.misc.transform import Transform from fontTools.pens.basePen import AbstractPen from fontTools.pens.pointPen import ( diff --git a/src/ufoLib2/objects/guideline.py b/src/ufoLib2/objects/guideline.py index 75dd434e..2697b459 100644 --- a/src/ufoLib2/objects/guideline.py +++ b/src/ufoLib2/objects/guideline.py @@ -2,7 +2,7 @@ from typing import Optional -from attr import define +from attrs import define from ufoLib2.objects.misc import AttrDictMixin from ufoLib2.serde import serde diff --git a/src/ufoLib2/objects/image.py b/src/ufoLib2/objects/image.py index 2aa08212..1de0de4f 100644 --- a/src/ufoLib2/objects/image.py +++ b/src/ufoLib2/objects/image.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, ClassVar, Iterator, Optional, Tuple -from attr import define, field +from attrs import define, field from fontTools.misc.transform import Identity, Transform from ufoLib2.serde import serde diff --git a/src/ufoLib2/objects/info/__init__.py b/src/ufoLib2/objects/info/__init__.py index bcae6898..f8201593 100644 --- a/src/ufoLib2/objects/info/__init__.py +++ b/src/ufoLib2/objects/info/__init__.py @@ -4,8 +4,8 @@ from functools import partial from typing import Any, Callable, List, Mapping, Optional, Sequence, TypeVar -import attr -from attr import define, field +import attrs +from attrs import define, field from fontTools.ufoLib import UFOReader from ufoLib2.objects.guideline import Guideline @@ -60,7 +60,7 @@ def _positive(instance: Any, attribute: Any, value: int) -> None: ) -_optional_positive = attr.validators.optional(_positive) +_optional_positive = attrs.validators.optional(_positive) # or maybe use IntFlag? diff --git a/src/ufoLib2/objects/info/woff.py b/src/ufoLib2/objects/info/woff.py index f13b64b4..ac40e37a 100644 --- a/src/ufoLib2/objects/info/woff.py +++ b/src/ufoLib2/objects/info/woff.py @@ -6,7 +6,7 @@ from typing import Any, List, Mapping, Optional, Sequence, Type, TypeVar -from attr import Attribute, define, field +from attrs import Attribute, define, field from ufoLib2.objects.misc import AttrDictMixin diff --git a/src/ufoLib2/objects/kerning.py b/src/ufoLib2/objects/kerning.py index e183dd54..cfefa8aa 100644 --- a/src/ufoLib2/objects/kerning.py +++ b/src/ufoLib2/objects/kerning.py @@ -7,7 +7,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import Converter + from cattrs import Converter KerningPair = Tuple[str, str] diff --git a/src/ufoLib2/objects/layer.py b/src/ufoLib2/objects/layer.py index c873c2eb..1d63da75 100644 --- a/src/ufoLib2/objects/layer.py +++ b/src/ufoLib2/objects/layer.py @@ -12,7 +12,7 @@ overload, ) -from attr import define, field +from attrs import define, field from fontTools.ufoLib.glifLib import GlyphSet from ufoLib2.constants import DEFAULT_LAYER_NAME @@ -30,7 +30,7 @@ from ufoLib2.typing import T if TYPE_CHECKING: - from cattr import Converter + from cattrs import Converter _GLYPH_NOT_LOADED = Glyph(name="___UFOLIB2_LAZY_GLYPH___") diff --git a/src/ufoLib2/objects/layerSet.py b/src/ufoLib2/objects/layerSet.py index e033cbdd..0d5ca63c 100644 --- a/src/ufoLib2/objects/layerSet.py +++ b/src/ufoLib2/objects/layerSet.py @@ -11,7 +11,7 @@ Sized, ) -from attr import define, field +from attrs import define, field from fontTools.ufoLib import UFOReader, UFOWriter from ufoLib2.constants import DEFAULT_LAYER_NAME @@ -28,7 +28,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import Converter + from cattrs import Converter _LAYER_NOT_LOADED = Layer(name="___UFOLIB2_LAZY_LAYER___") diff --git a/src/ufoLib2/objects/lib.py b/src/ufoLib2/objects/lib.py index 23b54d60..6fb98c56 100644 --- a/src/ufoLib2/objects/lib.py +++ b/src/ufoLib2/objects/lib.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from typing import Type - from cattr import Converter + from cattrs import Converter # unfortunately mypy is not smart enough to support recursive types like plist... # PlistEncodable = Union[ diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index eb092cc7..48df90b0 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -21,8 +21,8 @@ cast, ) -import attr -from attr import define, field +import attrs +from attrs import define, field from fontTools.misc.arrayTools import unionRect from fontTools.misc.transform import Transform from fontTools.pens.boundsPen import BoundsPen, ControlBoundsPen @@ -32,7 +32,7 @@ from ufoLib2.typing import Drawable, GlyphSet, HasIdentifier if TYPE_CHECKING: - from cattr import Converter + from cattrs import Converter class BoundingBox(NamedTuple): @@ -78,7 +78,7 @@ def _deepcopy_unlazify_attrs(self: Any, memo: Any) -> Any: (a.name if a.name[0] != "_" else a.name[1:]): deepcopy( getattr(self, a.name), memo ) - for a in attr.fields(self.__class__) + for a in attrs.fields(self.__class__) if a.init }, ) @@ -89,7 +89,7 @@ def _getstate_unlazify_attrs(self: Any) -> Dict[str, Any]: self.unlazify() return { a.name: getattr(self, a.name) if a.init else a.default - for a in attr.fields(self.__class__) + for a in attrs.fields(self.__class__) } @@ -100,8 +100,8 @@ def _getstate_unlazify_attrs(self: Any) -> Dict[str, Any]: # Below is adapted from `attrs._make._ClassBuilder._make_getstate_setstate` method: # https://github.com/python-attrs/attrs/blob/36ed0204/src/attr/_make.py#L931-L937 def _setstate_attrs(self: Any, state: Dict[str, Any]) -> None: - _bound_setattr = _obj_setattr.__get__(self, attr.Attribute) # type: ignore - for a in attr.fields(self.__class__): + _bound_setattr = _obj_setattr.__get__(self, attrs.Attribute) # type: ignore + for a in attrs.fields(self.__class__): if a.name in state: _bound_setattr(a.name, state[a.name]) @@ -373,7 +373,7 @@ class AttrDictMixin(AttrDictMixinMapping): @lru_cache(maxsize=None) def _key_to_attr_map(cls, reverse: bool = False) -> dict[str, str]: result = {} - for a in attr.fields(cls): + for a in attrs.fields(cls): attr_name = a.name key = attr_name if "rename_attr" in a.metadata: @@ -396,7 +396,7 @@ def __getitem__(self, key: str) -> Any: def __iter__(self) -> Iterator[str]: key_map = self._key_to_attr_map(reverse=True) - for attr_name in attr.fields_dict(self.__class__): + for attr_name in attrs.fields_dict(self.__class__): if getattr(self, attr_name) is not None: yield key_map[attr_name] diff --git a/src/ufoLib2/objects/point.py b/src/ufoLib2/objects/point.py index e9ffcf9f..8e3f746c 100644 --- a/src/ufoLib2/objects/point.py +++ b/src/ufoLib2/objects/point.py @@ -2,7 +2,7 @@ from typing import Optional -from attr import define +from attrs import define from ufoLib2.serde import serde diff --git a/tests/serde/test_json.py b/tests/serde/test_json.py index dd47bef8..954d86cf 100644 --- a/tests/serde/test_json.py +++ b/tests/serde/test_json.py @@ -9,7 +9,7 @@ import ufoLib2.objects # isort: off -pytest.importorskip("cattr") +pytest.importorskip("cattrs") import ufoLib2.serde.json # noqa: E402 diff --git a/tests/serde/test_msgpack.py b/tests/serde/test_msgpack.py index 6a697015..886582f5 100644 --- a/tests/serde/test_msgpack.py +++ b/tests/serde/test_msgpack.py @@ -6,7 +6,7 @@ import ufoLib2.objects # isort: off -pytest.importorskip("cattr") +pytest.importorskip("cattrs") import ufoLib2.serde.msgpack # noqa: E402 diff --git a/tests/serde/test_serde.py b/tests/serde/test_serde.py index d34442cb..aaf0cced 100644 --- a/tests/serde/test_serde.py +++ b/tests/serde/test_serde.py @@ -63,7 +63,7 @@ class Foo: def test_serde_all_objects(fmt: str, object_info: Dict[str, Any]) -> None: if fmt in ("json", "msgpack"): # skip these format tests if cattrs is not installed - pytest.importorskip("cattr") + pytest.importorskip("cattrs") klass = getattr(ufoLib2.objects, object_info["class_name"]) loads = getattr(klass, f"{fmt}_loads") diff --git a/tests/test_converters.py b/tests/test_converters.py index 0e55f000..dca1e3ef 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -48,7 +48,7 @@ ) # isort: off -cattr = pytest.importorskip("cattr") +cattrs = pytest.importorskip("cattrs") from ufoLib2.converters import register_hooks, structure, unstructure # noqa: E402 @@ -517,7 +517,7 @@ def test_unstructure_lazy_font(ufo_UbuTestData: Font) -> None: @pytest.mark.parametrize("forbid_extra_keys", [True, False]) def test_structure_forbid_extra_keys(forbid_extra_keys: bool) -> None: - conv = cattr.Converter(forbid_extra_keys=forbid_extra_keys) + conv = cattrs.Converter(forbid_extra_keys=forbid_extra_keys) register_hooks(conv) data = {"name": "a", "foo": "bar"} if forbid_extra_keys: @@ -599,7 +599,7 @@ def test_structure_forbid_extra_keys(forbid_extra_keys: bool) -> None: ], ) def test_omit_if_default(obj: Any, expected: Any, omit_if_default: bool) -> None: - conv = cattr.Converter(omit_if_default=omit_if_default) + conv = cattrs.Converter(omit_if_default=omit_if_default) register_hooks(conv) assert conv.unstructure(obj) == expected @@ -653,7 +653,7 @@ def test_omit_if_default(obj: Any, expected: Any, omit_if_default: bool) -> None ], ) def test_allow_bytes(obj: Any, expected: Any, allow_bytes: bool) -> None: - conv = cattr.Converter() + conv = cattrs.Converter() register_hooks(conv, allow_bytes=allow_bytes) assert conv.unstructure(obj) == expected @@ -661,7 +661,7 @@ def test_allow_bytes(obj: Any, expected: Any, allow_bytes: bool) -> None: def test_custom_type_overrides() -> None: - conv = cattr.Converter(type_overrides={Image: cattr.override(omit=True)}) + conv = cattrs.Converter(type_overrides={Image: cattrs.override(omit=True)}) register_hooks(conv) # check that Glyph.image attribute (of type Image) is omitted From 7c7f7f769177e51dd71038576e544b82bf15097f Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 3 Nov 2022 19:11:19 +0000 Subject: [PATCH 16/22] converters: pass on new detailed_validation option, fix test --- src/ufoLib2/converters.py | 4 +++- tests/test_converters.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ufoLib2/converters.py b/src/ufoLib2/converters.py index d6ced211..d01e16e1 100644 --- a/src/ufoLib2/converters.py +++ b/src/ufoLib2/converters.py @@ -59,7 +59,9 @@ def attrs_hook_factory( attribs = fields(base) # PEP563 postponed annotations need resolving as we check Attribute.type below resolve_types(base) - kwargs: dict[str, bool | AttributeOverride] = {} + kwargs: dict[str, bool | AttributeOverride] = { + "_cattrs_detailed_validation": conv.detailed_validation + } if structuring: kwargs["_cattrs_forbid_extra_keys"] = conv.forbid_extra_keys kwargs["_cattrs_prefer_attrib_converters"] = conv._prefer_attrib_converters diff --git a/tests/test_converters.py b/tests/test_converters.py index dca1e3ef..047a43e7 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -517,11 +517,17 @@ def test_unstructure_lazy_font(ufo_UbuTestData: Font) -> None: @pytest.mark.parametrize("forbid_extra_keys", [True, False]) def test_structure_forbid_extra_keys(forbid_extra_keys: bool) -> None: - conv = cattrs.Converter(forbid_extra_keys=forbid_extra_keys) + conv = cattrs.Converter( + forbid_extra_keys=forbid_extra_keys, + detailed_validation=False, + ) register_hooks(conv) data = {"name": "a", "foo": "bar"} if forbid_extra_keys: - with pytest.raises(Exception, match="Extra fields in constructor for .*: foo"): + with pytest.raises( + cattrs.errors.ForbiddenExtraKeysError, + match="Extra fields in constructor for .*: foo", + ): conv.structure(data, Glyph) else: assert conv.structure(data, Glyph) == Glyph(name="a") From 12068730d04133479687715870eb61f28f35f27a Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Thu, 3 Nov 2022 19:14:15 +0000 Subject: [PATCH 17/22] recompile requirements-dev.txt as well --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4b4b1402..79c4abe1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.11 # To update, run: # # pip-compile --output-file=requirements-dev.txt requirements-dev.in # -attrs==21.4.0 +attrs==22.1.0 # via # -c requirements.txt # pytest From ebc3131c52d0b2da84df7857cf33202baa0a6542 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 4 Nov 2022 12:07:53 +0000 Subject: [PATCH 18/22] make mypy happy again --- src/ufoLib2/converters.py | 6 +++--- src/ufoLib2/objects/misc.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/ufoLib2/converters.py b/src/ufoLib2/converters.py index d01e16e1..1c852ae5 100644 --- a/src/ufoLib2/converters.py +++ b/src/ufoLib2/converters.py @@ -51,8 +51,8 @@ def is_ufoLib2_class_with_custom_structure(cls: Type[Any]) -> bool: def register_hooks(conv: Converter, allow_bytes: bool = True) -> None: def attrs_hook_factory( - cls: Type[Any], gen_fn: Callable[..., Callable[[Any], Any]], structuring: bool - ) -> Callable[[Any], Any]: + cls: Type[Any], gen_fn: Callable[..., Callable[..., Any]], structuring: bool + ) -> Callable[..., Any]: base = get_origin(cls) if base is None: base = cls @@ -96,7 +96,7 @@ def attrs_hook_factory( def custom_unstructure_hook_factory(cls: Type[Any]) -> Callable[[Any], Any]: return partial(cls._unstructure, converter=conv) - def custom_structure_hook_factory(cls: Type[Any]) -> Callable[[Any], Any]: + def custom_structure_hook_factory(cls: Type[Any]) -> Callable[[Any, Any], Any]: return partial(cls._structure, converter=conv) def unstructure_transform(t: Transform) -> Tuple[float]: diff --git a/src/ufoLib2/objects/misc.py b/src/ufoLib2/objects/misc.py index 48df90b0..6c2f0b9f 100644 --- a/src/ufoLib2/objects/misc.py +++ b/src/ufoLib2/objects/misc.py @@ -15,7 +15,6 @@ Optional, Sequence, Set, - Tuple, Type, TypeVar, cast, @@ -32,6 +31,10 @@ from ufoLib2.typing import Drawable, GlyphSet, HasIdentifier if TYPE_CHECKING: + # Importing 'AttrsInstance' from 'attr' instead of 'attrs' namespace because + # v22.1.0 is missing the symbol: https://github.com/python-attrs/attrs/issues/987 + # from attrs import AttrsInstance + from attr import AttrsInstance from cattrs import Converter @@ -371,7 +374,9 @@ class AttrDictMixin(AttrDictMixinMapping): @classmethod @lru_cache(maxsize=None) - def _key_to_attr_map(cls, reverse: bool = False) -> dict[str, str]: + def _key_to_attr_map( + cls: Type[AttrsInstance], reverse: bool = False + ) -> dict[str, str]: result = {} for a in attrs.fields(cls): attr_name = a.name @@ -396,7 +401,8 @@ def __getitem__(self, key: str) -> Any: def __iter__(self) -> Iterator[str]: key_map = self._key_to_attr_map(reverse=True) - for attr_name in attrs.fields_dict(self.__class__): + cls = cast("Type[AttrsInstance]", self.__class__) + for attr_name in attrs.fields_dict(cls): if getattr(self, attr_name) is not None: yield key_map[attr_name] From b88ff098a6409d5d5557542b9bf569bca6ec1751 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 4 Nov 2022 12:11:16 +0000 Subject: [PATCH 19/22] do use orjson typing stubs after all --- src/ufoLib2/serde/json.py | 20 ++++++++++++-------- tests/serde/test_json.py | 2 -- tox.ini | 2 -- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ufoLib2/serde/json.py b/src/ufoLib2/serde/json.py index b6e27565..9a99e0a1 100644 --- a/src/ufoLib2/serde/json.py +++ b/src/ufoLib2/serde/json.py @@ -1,17 +1,18 @@ from __future__ import annotations -from typing import Any, Type, cast +import json +from typing import Any, Type from ufoLib2.converters import structure, unstructure from ufoLib2.typing import T have_orjson = False try: - import orjson as json # type: ignore + import orjson have_orjson = True except ImportError: - import json # type: ignore + pass def dumps( @@ -26,18 +27,21 @@ def dumps( if indent is not None: if indent != 2: raise ValueError("indent must be 2 or None for orjson") - kwargs["option"] = kwargs.pop("option", 0) | json.OPT_INDENT_2 + kwargs["option"] = kwargs.pop("option", 0) | orjson.OPT_INDENT_2 if sort_keys: - kwargs["option"] = kwargs.pop("option", 0) | json.OPT_SORT_KEYS + kwargs["option"] = kwargs.pop("option", 0) | orjson.OPT_SORT_KEYS # orjson.dumps always returns bytes - result = json.dumps(data, **kwargs) + result = orjson.dumps(data, **kwargs) else: # built-in json.dumps returns a string, not bytes, hence the encoding s = json.dumps(data, indent=indent, sort_keys=sort_keys, **kwargs) result = s.encode("utf-8") - return cast(bytes, result) + return result def loads(s: str | bytes, object_class: Type[T], **kwargs: Any) -> T: - data = json.loads(s, **kwargs) + if have_orjson: + data = orjson.loads(s, **kwargs) + else: + data = json.loads(s, **kwargs) return structure(data, object_class) diff --git a/tests/serde/test_json.py b/tests/serde/test_json.py index 954d86cf..67cdab8b 100644 --- a/tests/serde/test_json.py +++ b/tests/serde/test_json.py @@ -20,7 +20,6 @@ def test_dumps_loads( ) -> None: if not have_orjson: monkeypatch.setattr(ufoLib2.serde.json, "have_orjson", have_orjson) - monkeypatch.setattr(ufoLib2.serde.json, "json", json) font = ufo_UbuTestData data = font.json_dumps() # type: ignore @@ -52,7 +51,6 @@ def test_dump_load( ) -> None: if not have_orjson: monkeypatch.setattr(ufoLib2.serde.json, "have_orjson", have_orjson) - monkeypatch.setattr(ufoLib2.serde.json, "json", json) font = ufo_UbuTestData with open(tmp_path / "test.json", "wb") as f: diff --git a/tox.ini b/tox.ini index 8d418c3e..e55a83b3 100644 --- a/tox.ini +++ b/tox.ini @@ -28,8 +28,6 @@ deps = commands = black --check --diff . isort --skip-gitignore --check-only --diff src tests - # typing stubs for orjson exist but I don't want mypy to use them! - pip uninstall -y orjson mypy --strict src tests flake8 From c6e188c2b2a5c4ca60394e66c89be5df9307df77 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 4 Nov 2022 15:28:42 +0000 Subject: [PATCH 20/22] add comment to test_pickle_lazy_font [skip ci] --- tests/objects/test_font.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/objects/test_font.py b/tests/objects/test_font.py index 6a7d49a1..b4022542 100644 --- a/tests/objects/test_font.py +++ b/tests/objects/test_font.py @@ -114,6 +114,8 @@ def test_pickle_lazy_font(datadir: Path, lazy: Optional[bool]) -> None: if lazy is not None: font = Font.open(datadir / "UbuTestData.ufo", lazy=lazy) else: + # lazy is None by default for a Font that is not opened from a file + # but created from scratch font = Font() assert lazy is font._lazy From 36d4632697de0f26fd4b52c7313a2bae7ad7cb29 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 4 Nov 2022 17:40:06 +0000 Subject: [PATCH 21/22] add standalone load/dump functions in each ufoLib2.serde submodule So one can do ufoLib2.serde.json.load('MyFont.json', ufoLib2.Font) in alternative to ufoLib2.Font.json_load('MyFont.json') --- src/ufoLib2/serde/__init__.py | 63 +++++++++-------------------------- src/ufoLib2/serde/json.py | 19 +++++++++-- src/ufoLib2/serde/msgpack.py | 13 ++++++-- src/ufoLib2/serde/util.py | 25 ++++++++++++++ 4 files changed, 69 insertions(+), 51 deletions(-) create mode 100644 src/ufoLib2/serde/util.py diff --git a/src/ufoLib2/serde/__init__.py b/src/ufoLib2/serde/__init__.py index 6a9b21d1..0573abb8 100644 --- a/src/ufoLib2/serde/__init__.py +++ b/src/ufoLib2/serde/__init__.py @@ -2,52 +2,31 @@ from functools import partialmethod from importlib import import_module -from typing import IO, Any, AnyStr, BinaryIO, Callable, Type, cast +from typing import IO, Any, AnyStr, Callable, Type from ufoLib2.typing import PathLike, T _SERDE_FORMATS_ = ("json", "msgpack") +# the @serde decorator sets these as @classmethods on the class, hence +# the cls argument must appear as the first argument; in the standalone loads/load +# functions the input string or file-like object is the first argument and the second +# argument is the object_class, so below just swap the order of the arguments def _loads( - cls: Type[Any], s: str | bytes, *, __serde_submodule: Any, **kwargs: Any -) -> Any: - return __serde_submodule.loads(s, cls, **kwargs) + cls: Type[T], s: str | bytes, *, __callback: Callable[..., T], **kwargs: Any +) -> T: + return __callback(s, cls, **kwargs) def _load( - cls: Type[Any], + cls: Type[T], fp: PathLike | IO[AnyStr], *, - __loads_method: Callable[..., Any], + __callback: Callable[..., T], **kwargs: Any, -) -> Any: - data: str | bytes - if hasattr(fp, "read"): - fp = cast(IO[AnyStr], fp) - data = fp.read() - else: - fp = cast(PathLike, fp) - with open(fp, "rb") as f: - data = f.read() - return __loads_method(data, **kwargs) - - -def _dumps(self: Any, *, __serde_submodule: Any, **kwargs: Any) -> Any: - return __serde_submodule.dumps(self, **kwargs) - - -def _dump( - self: Any, fp: PathLike | BinaryIO, *, __dumps_method_name: str, **kwargs: Any -) -> None: - data: bytes = getattr(self, __dumps_method_name)(**kwargs) - if hasattr(fp, "write"): - fp = cast(BinaryIO, fp) - fp.write(data) - else: - fp = cast(PathLike, fp) - with open(fp, "wb") as f: - f.write(data) +) -> T: + return __callback(fp, cls, **kwargs) def serde(cls: Type[T]) -> Type[T]: @@ -108,25 +87,15 @@ def raise_error(*args: Any, **kwargs: Any) -> None: setattr( cls, f"{fmt}_loads", - partialmethod(classmethod(_loads), __serde_submodule=serde_submodule), + partialmethod(classmethod(_loads), __callback=serde_submodule.loads), ) setattr( cls, f"{fmt}_load", - partialmethod( - classmethod(_load), __loads_method=getattr(cls, f"{fmt}_loads") - ), - ) - setattr( - cls, - f"{fmt}_dumps", - partialmethod(_dumps, __serde_submodule=serde_submodule), - ) - setattr( - cls, - f"{fmt}_dump", - partialmethod(_dump, __dumps_method_name=f"{fmt}_dumps"), + partialmethod(classmethod(_load), __callback=serde_submodule.load), ) + setattr(cls, f"{fmt}_dumps", serde_submodule.dumps) + setattr(cls, f"{fmt}_dump", serde_submodule.dump) supported_formats.append(fmt) setattr(cls, "_SERDE_FORMATS_", tuple(supported_formats)) diff --git a/src/ufoLib2/serde/json.py b/src/ufoLib2/serde/json.py index 9a99e0a1..37472a39 100644 --- a/src/ufoLib2/serde/json.py +++ b/src/ufoLib2/serde/json.py @@ -1,10 +1,11 @@ from __future__ import annotations import json -from typing import Any, Type +from typing import Any, BinaryIO, Type from ufoLib2.converters import structure, unstructure -from ufoLib2.typing import T +from ufoLib2.serde.util import read_bytes, write_bytes +from ufoLib2.typing import PathLike, T have_orjson = False try: @@ -45,3 +46,17 @@ def loads(s: str | bytes, object_class: Type[T], **kwargs: Any) -> T: else: data = json.loads(s, **kwargs) return structure(data, object_class) + + +def dump( + obj: Any, + fp: PathLike | BinaryIO, + indent: int | None = None, + sort_keys: bool = False, + **kwargs: Any, +) -> None: + write_bytes(fp, dumps(obj, indent=indent, sort_keys=sort_keys, **kwargs)) + + +def load(fp: PathLike | BinaryIO, object_class: Type[T], **kwargs: Any) -> T: + return loads(read_bytes(fp), object_class, **kwargs) diff --git a/src/ufoLib2/serde/msgpack.py b/src/ufoLib2/serde/msgpack.py index 29951659..c643a2f8 100644 --- a/src/ufoLib2/serde/msgpack.py +++ b/src/ufoLib2/serde/msgpack.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import Any, Type, cast +from typing import Any, BinaryIO, Type, cast import msgpack # type: ignore from ufoLib2.converters import binary_converter -from ufoLib2.typing import T +from ufoLib2.serde.util import read_bytes, write_bytes +from ufoLib2.typing import PathLike, T def dumps(obj: Any, **kwargs: Any) -> bytes: @@ -17,3 +18,11 @@ def dumps(obj: Any, **kwargs: Any) -> bytes: def loads(s: bytes, object_class: Type[T], **kwargs: Any) -> T: data = msgpack.unpackb(s, **kwargs) return binary_converter.structure(data, object_class) + + +def dump(obj: Any, fp: PathLike | BinaryIO, **kwargs: Any) -> None: + write_bytes(fp, dumps(obj, **kwargs)) + + +def load(fp: PathLike | BinaryIO, object_class: Type[T], **kwargs: Any) -> T: + return loads(read_bytes(fp), object_class, **kwargs) diff --git a/src/ufoLib2/serde/util.py b/src/ufoLib2/serde/util.py new file mode 100644 index 00000000..dff70624 --- /dev/null +++ b/src/ufoLib2/serde/util.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import BinaryIO, cast + +from ufoLib2.typing import PathLike + + +def read_bytes(fp: PathLike | BinaryIO) -> bytes: + if hasattr(fp, "read"): + fp = cast(BinaryIO, fp) + return fp.read() + else: + fp = cast(PathLike, fp) + with open(fp, "rb") as f: + return f.read() + + +def write_bytes(fp: PathLike | BinaryIO, data: bytes) -> None: + if hasattr(fp, "write"): + fp = cast(BinaryIO, fp) + fp.write(data) + else: + fp = cast(PathLike, fp) + with open(fp, "wb") as f: + f.write(data) From 7a9d1debe69be1b0beac67470c018cf989436262 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Fri, 4 Nov 2022 18:01:37 +0000 Subject: [PATCH 22/22] fix typos in comments [skip ci] --- tests/serde/test_serde.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/serde/test_serde.py b/tests/serde/test_serde.py index aaf0cced..f45e210e 100644 --- a/tests/serde/test_serde.py +++ b/tests/serde/test_serde.py @@ -10,7 +10,7 @@ def test_raise_import_error(monkeypatch: Any) -> None: - # pretent we can't import the module (e.g. msgpack not installed) + # pretend we can't import the module (e.g. msgpack not installed) monkeypatch.setitem(sys.modules, "ufoLib2.serde.msgpack", None) with pytest.raises(ImportError, match="ufoLib2.serde.msgpack"): @@ -25,7 +25,7 @@ class Foo: foo = Foo(1) with pytest.raises(ImportError, match="ufoLib2.serde.msgpack"): - # since the method is only added dynamicall at runtime, mypy complains that + # since the method is only added dynamically at runtime, mypy complains that # "Foo" has no attribute "msgpack_dumps" -- so I shut it up foo.msgpack_dumps() # type: ignore