From 3ffe88e681f54f43a9ccad6145cdb3ae16114137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Thu, 24 Jun 2021 09:48:13 +0100 Subject: [PATCH] Handle YANG binary type As described in RFC7959, section 9.8.2, values of the ``binary`` built-in type are represented with the base64 encoding scheme. Encode / decode from that scheme on serialise/deserialise, and use native ``bytes`` type for Python representation. --- docs/yang.md | 2 +- pyangbind/lib/serialise.py | 7 +- pyangbind/lib/yangtypes.py | 36 +++++++--- pyangbind/plugin/pybind.py | 7 +- requirements.txt | 1 - tests/binary/binary.yang | 2 +- tests/binary/run.py | 67 +++++++------------ .../json/complete-obj.json | 2 +- tests/serialise/ietf-json-deserialise/run.py | 4 +- .../ietf-json-serialise/json/obj.json | 2 +- tests/serialise/ietf-json-serialise/run.py | 3 +- .../json-deserialise/json/alltypes.json | 2 +- tests/serialise/json-deserialise/run.py | 3 +- .../json-serialise/json/expected-output.json | 2 +- tests/serialise/json-serialise/run.py | 3 +- tests/serialise/roundtrip/run.py | 4 -- tests/serialise/xml-deserialise/xml/obj.xml | 2 +- tests/serialise/xml-serialise/run.py | 3 +- tests/serialise/xml-serialise/xml/obj.xml | 2 +- tests/union/run.py | 3 +- tox.ini | 3 - 21 files changed, 74 insertions(+), 86 deletions(-) diff --git a/docs/yang.md b/docs/yang.md index 1cec67eb..a0741645 100644 --- a/docs/yang.md +++ b/docs/yang.md @@ -52,7 +52,7 @@ PyangBind does not currently try and be feature complete against the YANG langua **Type** | **Sub-Statement** | **Supported Type** | **Unit Tests** --------------------|--------------------|--------------------------|--------------- - **binary** | - | [bitarray](https://github.com/ilanschnell/bitarray) | tests/binary + **binary** | - | [bytes](https://docs.python.org/3/library/stdtypes.html?#bytes) | tests/binary - | length | Supported | tests/binary **bits** | - | Not supported | N/A - | position | Not supported | N/A diff --git a/pyangbind/lib/serialise.py b/pyangbind/lib/serialise.py index 3d4c2c43..02c13fa3 100644 --- a/pyangbind/lib/serialise.py +++ b/pyangbind/lib/serialise.py @@ -28,6 +28,7 @@ import json from collections import OrderedDict from decimal import Decimal +import base64 import six from enum import IntEnum @@ -130,7 +131,7 @@ def default(self, obj): elif orig_yangt in ["string", "enumeration"]: return six.text_type(obj) elif orig_yangt in ["binary"]: - return obj.to01() + return six.text_type(base64.b64encode(obj), "ascii") elif orig_yangt in ["decimal64"]: return self.yangt_decimal(obj) elif orig_yangt in ["bool"]: @@ -180,8 +181,8 @@ def map_pyangbind_type(self, map_val, original_yang_type, obj): elif map_val in ["pyangbind.lib.yangtypes.RestrictedPrecisionDecimal", "RestrictedPrecisionDecimal"]: # NOTE: this doesn't seem like it needs to be a special case? return self.yangt_decimal(obj) - elif map_val in ["bitarray.bitarray"]: - return obj.to01() + elif map_val in ["pyangbind.lib.yangtypes.YANGBinary", "YANGBinary"]: + return six.text_type(base64.b64encode(obj), "ascii") elif map_val in ["unicode"]: return six.text_type(obj) elif map_val in ["pyangbind.lib.yangtypes.YANGBool"]: diff --git a/pyangbind/lib/yangtypes.py b/pyangbind/lib/yangtypes.py index e9b4fb9c..5e3fca82 100644 --- a/pyangbind/lib/yangtypes.py +++ b/pyangbind/lib/yangtypes.py @@ -21,6 +21,7 @@ """ from __future__ import unicode_literals +import base64 import collections from collections import abc import copy @@ -29,7 +30,6 @@ import regex import six -from bitarray import bitarray # Words that could turn up in YANG definition files that are actually # reserved names in Python, such as being builtin types. This list is @@ -274,9 +274,7 @@ def build_length_range_tuples(range, length=False, multiplier=1): def in_range_check(low_high_tuples, length=False): def range_check(value): - if length and isinstance(value, bitarray): - value = value.length() - elif length: + if length: value = len(value) range_results = [] for check_tuple in low_high_tuples: @@ -334,12 +332,7 @@ def in_dictionary_check(dictionary): except Exception: raise TypeError("must specify a numeric type for a range " + "argument") elif rtype == "length": - # When the type is a binary then the length is specified in - # octets rather than bits, so we must specify the length to - # be multiplied by 8. multiplier = 1 - if base_type == bitarray: - multiplier = 8 lengths = [] for range_spec in rarg: lengths.append(build_length_range_tuples(range_spec, length=True, multiplier=multiplier)) @@ -1352,3 +1345,28 @@ def __str__(self): return str(self._get_ptr()) return type(ReferencePathType(*args, **kwargs)) + + +class YANGBinary(bytes): + """ + A custom binary class for using in YANG. + """ + + def __new__(self, *args, **kwargs): + value = b"" + if args: + value = args[0] + if isinstance(value, str): + value = base64.b64decode(value) + elif isinstance(value, bytes): + value = value + else: + raise ValueError(f"invalid type for {value}: {type(value)}") + + return bytes.__new__(self, value) + + def __repr__(self): + return str(self) + + def __str__(self, encoding="ascii", errors="replace"): + return str(self, encoding=encoding, errors=errors) diff --git a/pyangbind/plugin/pybind.py b/pyangbind/plugin/pybind.py index 2b364e73..5d8c58df 100644 --- a/pyangbind/plugin/pybind.py +++ b/pyangbind/plugin/pybind.py @@ -30,12 +30,11 @@ from collections import OrderedDict import six -from bitarray import bitarray from pyang import plugin, statements, util import pyangbind.helpers.misc as misc_help from pyangbind.helpers.identity import IdentityStore -from pyangbind.lib.yangtypes import RestrictedClassType, YANGBool, safe_name +from pyangbind.lib.yangtypes import RestrictedClassType, YANGBool, safe_name, YANGBinary # Python3 support if six.PY3: @@ -100,7 +99,7 @@ "quote_arg": True, "pytype": YANGBool, }, - "binary": {"native_type": "bitarray", "base_type": True, "quote_arg": True, "pytype": bitarray}, + "binary": {"native_type": "YANGBinary", "base_type": True, "quote_arg": True, "pytype": YANGBinary}, "uint8": { "native_type": ("RestrictedClassType(base_type=int," + " restriction_dict={'range': ['0..255']}, int_size=8)"), "base_type": True, @@ -317,13 +316,13 @@ def build_pybind(ctx, modules, fd): "YANGListType", "YANGDynClass", "ReferenceType", + "YANGBinary", ] for library in yangtypes_imports: ctx.pybind_common_hdr += "from pyangbind.lib.yangtypes import {}\n".format(library) ctx.pybind_common_hdr += "from pyangbind.lib.base import PybindBase\n" ctx.pybind_common_hdr += "from collections import OrderedDict\n" ctx.pybind_common_hdr += "from decimal import Decimal\n" - ctx.pybind_common_hdr += "from bitarray import bitarray\n" ctx.pybind_common_hdr += "import six\n" # Python3 support diff --git a/requirements.txt b/requirements.txt index cf8ceb9a..6ae06d55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ pyang -bitarray lxml regex six diff --git a/tests/binary/binary.yang b/tests/binary/binary.yang index 12850228..e67ab227 100644 --- a/tests/binary/binary.yang +++ b/tests/binary/binary.yang @@ -21,7 +21,7 @@ module binary { leaf b2 { type binary; - default "0100"; + default "eWFuZw=="; /* yang */ description "A test leaf with a default"; } diff --git a/tests/binary/run.py b/tests/binary/run.py index 5dbe7d1a..c9ce9bad 100755 --- a/tests/binary/run.py +++ b/tests/binary/run.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from bitarray import bitarray import unittest from tests.base import PyangBindTestCase @@ -19,8 +18,8 @@ def test_binary_leafs_exist(self): hasattr(self.binary_obj.container, leaf), "Element did not exist in container (%s)" % leaf ) - def test_set_bitarray_from_different_datatypes(self): - for value in [("01110", True), ({"42": 42}, True), (-42, False), ("Arthur Dent", False)]: + def test_set_binary_from_different_datatypes(self): + for value in [(b"42", True), ({"42": 42}, False), (-42, False), ("Arthur Dent", False)]: with self.subTest(value=value): allowed = True try: @@ -30,46 +29,46 @@ def test_set_bitarray_from_different_datatypes(self): self.assertEqual(allowed, value[1], "Could incorrectly set b1 to %s" % value[0]) def test_binary_leaf_default_value(self): - default = bitarray("0100") + default = b"yang" self.assertEqual( self.binary_obj.container.b2._default, default, "Default for leaf b2 was not set correctly (%s != %s)" % (self.binary_obj.container.b2._default, default), ) - def test_binary_leaf_is_empty_bitarray_by_default(self): - empty = bitarray() + def test_binary_leaf_is_empty_by_default(self): + empty = b"" self.assertEqual( self.binary_obj.container.b2, empty, - "Value of bitarray was not null when checking b2 (%s != %s)" % (self.binary_obj.container.b2, empty), + "Value of binary leaf was not null when checking b2 (%s != %s)" % (self.binary_obj.container.b2, empty), ) def test_binary_leaf_is_not_changed_by_default(self): self.assertFalse( self.binary_obj.container.b2._changed(), - "Unset bitarray specified changed when was default (%s != False)" + "Unset binary leaf specified changed when was default (%s != False)" % self.binary_obj.container.b2._changed(), ) - def test_set_bitarray_stores_value(self): - bits = bitarray("010") + def test_set_binary_stores_value(self): + bits = b"010" self.binary_obj.container.b2 = bits self.assertEqual( self.binary_obj.container.b2, bits, - "Bitarray not successfully set (%s != %s)" % (self.binary_obj.container.b2, bits), + "Binary leaf not successfully set (%s != %s)" % (self.binary_obj.container.b2, bits), ) - def test_setting_bitarray_set_changed(self): - self.binary_obj.container.b2 = bitarray("010") + def test_setting_binary_set_changed(self): + self.binary_obj.container.b2 = b"010" self.assertTrue( self.binary_obj.container.b2._changed(), - "Bitarray value not flagged as changed (%s != True)" % self.binary_obj.container.b2._changed(), + "Binary leaf value not flagged as changed (%s != True)" % self.binary_obj.container.b2._changed(), ) - def test_set_specific_length_bitarray(self): - for bits in [("0", False), ("1000000011110000", True), ("111111110000000011111111", False)]: + def test_set_specific_length_binary_leaf(self): + for bits in [(b"1", False), (b"12", True), (b"1234", False)]: with self.subTest(bits=bits): allowed = True try: @@ -82,13 +81,8 @@ def test_set_specific_length_bitarray(self): "limited length binary incorrectly set to %s (%s != %s)" % (bits[0], bits[1], allowed), ) - def test_set_bitarray_with_length_range(self): - for bits in [ - ("0", False), - ("1111111100000000", True), - ("111111110000000011111111", True), - ("1111111100000000111111110000000011110000", False), - ]: + def test_set_binary_leaf_with_length_range(self): + for bits in [(b"1", False), (b"12", True), (b"1234", True), (b"12345", False)]: with self.subTest(bits=bits): allowed = True try: @@ -101,25 +95,16 @@ def test_set_bitarray_with_length_range(self): "Limited length binary with range incorrectly set to %s (%s != %s)" % (bits[0], bits[1], allowed), ) - def test_set_bitarray_with_complex_length(self): + def test_set_binary_leaf_with_complex_length(self): for bits in [ - ("0", False), - ("1111000011110000", True), - ("111100001111000011110000", True), - ("1111000011110000111100001111000011110000", False), - ("111100001111000011110000111100001111000011110000", True), - ("111100001111000011110000111100001111000011110000" "11110000111100001111000011110000", True), - ( - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "111100001111000011110000111100001111000011110000" - "1010101010101010", - False, - ), + (b"1", False), + (b"12", True), + (b"123", True), + (b"12345", False), + (b"123456", True), + (b"123456789_", True), + (b"123456789_123456789_123456789_123456789_12", True), + (b"123456789_123456789_123456789_123456789_123", False), ]: with self.subTest(bits=bits): allowed = True diff --git a/tests/serialise/ietf-json-deserialise/json/complete-obj.json b/tests/serialise/ietf-json-deserialise/json/complete-obj.json index bb33591c..d7078c8e 100644 --- a/tests/serialise/ietf-json-deserialise/json/complete-obj.json +++ b/tests/serialise/ietf-json-deserialise/json/complete-obj.json @@ -42,7 +42,7 @@ "one-leaf": "hi", "typedef-one": "test", "boolean": true, - "binary": "010101", + "binary": "eWFuZw==", "union": "16", "k1": 1, "enumeration": "one", diff --git a/tests/serialise/ietf-json-deserialise/run.py b/tests/serialise/ietf-json-deserialise/run.py index 0cd7e82c..cc1d8e87 100755 --- a/tests/serialise/ietf-json-deserialise/run.py +++ b/tests/serialise/ietf-json-deserialise/run.py @@ -7,8 +7,6 @@ from collections import OrderedDict from decimal import Decimal -from bitarray import bitarray - from pyangbind.lib.serialise import pybindJSONDecoder from tests.base import PyangBindTestCase @@ -58,7 +56,7 @@ def test_all_the_types(self): "one-leaf": "hi", "typedef-one": "test", "boolean": True, - "binary": bitarray("010101"), + "binary": b"yang", "union": "16", "identityref": "idone", "enumeration": "one", diff --git a/tests/serialise/ietf-json-serialise/json/obj.json b/tests/serialise/ietf-json-serialise/json/obj.json index a7aa39c6..43f83b83 100644 --- a/tests/serialise/ietf-json-serialise/json/obj.json +++ b/tests/serialise/ietf-json-serialise/json/obj.json @@ -42,7 +42,7 @@ "one-leaf": "hi", "typedef-one": "test", "boolean": true, - "binary": "010101", + "binary": "eWFuZw==", "union": "16", "k1": 1, "enumeration": "one", diff --git a/tests/serialise/ietf-json-serialise/run.py b/tests/serialise/ietf-json-serialise/run.py index fc832f61..6ea2c085 100755 --- a/tests/serialise/ietf-json-serialise/run.py +++ b/tests/serialise/ietf-json-serialise/run.py @@ -6,7 +6,6 @@ from decimal import Decimal import six -from bitarray import bitarray from pyangbind.lib.serialise import pybindIETFJSONEncoder from pyangbind.lib.xpathhelper import YANGPathHelper @@ -52,7 +51,7 @@ def test_serialise_full_container(self): self.serialise_obj.c1.t1.add(32) self.serialise_obj.c1.l1[1].leafref = 16 - self.serialise_obj.c1.l1[1].binary = bitarray("010101") + self.serialise_obj.c1.l1[1].binary = b"yang" self.serialise_obj.c1.l1[1].boolean = True self.serialise_obj.c1.l1[1].enumeration = "one" self.serialise_obj.c1.l1[1].identityref = "idone" diff --git a/tests/serialise/json-deserialise/json/alltypes.json b/tests/serialise/json-deserialise/json/alltypes.json index 1dec6330..f4ced607 100644 --- a/tests/serialise/json-deserialise/json/alltypes.json +++ b/tests/serialise/json-deserialise/json/alltypes.json @@ -13,7 +13,7 @@ "one-leaf": "hi", "typedef-one": "test", "boolean": true, - "binary": "010101", + "binary": "eWFuZw==", "union": "16", "k1": 1, "enumeration": "one", diff --git a/tests/serialise/json-deserialise/run.py b/tests/serialise/json-deserialise/run.py index 0aae7f0d..abf1b22c 100755 --- a/tests/serialise/json-deserialise/run.py +++ b/tests/serialise/json-deserialise/run.py @@ -4,7 +4,6 @@ import json import os.path import unittest -from bitarray import bitarray from decimal import Decimal import pyangbind.lib.pybindJSON as pbJ @@ -60,7 +59,7 @@ def test_all_the_types(self): "one-leaf": "hi", "typedef-one": "test", "boolean": True, - "binary": bitarray("010101"), + "binary": b"yang", "union": "16", "identityref": "idone", "enumeration": "one", diff --git a/tests/serialise/json-serialise/json/expected-output.json b/tests/serialise/json-serialise/json/expected-output.json index c78ab0f9..2c8e1449 100644 --- a/tests/serialise/json-serialise/json/expected-output.json +++ b/tests/serialise/json-serialise/json/expected-output.json @@ -51,7 +51,7 @@ "one-leaf": "hi", "typedef-one": "test", "boolean": true, - "binary": "010101", + "binary": "eWFuZw==", "union": "16", "k1": 1, "enumeration": "one", diff --git a/tests/serialise/json-serialise/run.py b/tests/serialise/json-serialise/run.py index abe3a6fd..bb633719 100755 --- a/tests/serialise/json-serialise/run.py +++ b/tests/serialise/json-serialise/run.py @@ -6,7 +6,6 @@ from decimal import Decimal import six -from bitarray import bitarray from pyangbind.lib.pybindJSON import dumps from pyangbind.lib.xpathhelper import YANGPathHelper @@ -48,7 +47,7 @@ def test_full_serialise(self): self.serialise_obj.c1.t1.add(32) self.serialise_obj.c1.l1[1].leafref = 16 - self.serialise_obj.c1.l1[1].binary = bitarray("010101") + self.serialise_obj.c1.l1[1].binary = b"yang" self.serialise_obj.c1.l1[1].boolean = True self.serialise_obj.c1.l1[1].enumeration = "one" self.serialise_obj.c1.l1[1].identityref = "idone" diff --git a/tests/serialise/roundtrip/run.py b/tests/serialise/roundtrip/run.py index 4507d4c1..0802973d 100755 --- a/tests/serialise/roundtrip/run.py +++ b/tests/serialise/roundtrip/run.py @@ -2,14 +2,10 @@ from __future__ import unicode_literals import json -import os.path import unittest -from bitarray import bitarray -from decimal import Decimal import pyangbind.lib.pybindJSON as pbJ import pyangbind.lib.serialise as pbS -from pyangbind.lib.serialise import pybindJSONDecoder from pyangbind.lib.xpathhelper import YANGPathHelper from tests.base import PyangBindTestCase diff --git a/tests/serialise/xml-deserialise/xml/obj.xml b/tests/serialise/xml-deserialise/xml/obj.xml index af2d4e16..96a16860 100644 --- a/tests/serialise/xml-deserialise/xml/obj.xml +++ b/tests/serialise/xml-deserialise/xml/obj.xml @@ -20,7 +20,7 @@ chicken -100 16 - 010101 + eWFuZw== true one idone diff --git a/tests/serialise/xml-serialise/run.py b/tests/serialise/xml-serialise/run.py index b095c096..3629d7da 100755 --- a/tests/serialise/xml-serialise/run.py +++ b/tests/serialise/xml-serialise/run.py @@ -5,7 +5,6 @@ from decimal import Decimal import six -from bitarray import bitarray from lxml import objectify from pyangbind.lib.serialise import pybindIETFXMLEncoder @@ -41,7 +40,7 @@ def test_serialise_full_container(self): self.serialise_obj.c1.t1.add(32) self.serialise_obj.c1.l1[1].leafref = 16 - self.serialise_obj.c1.l1[1].binary = bitarray("010101") + self.serialise_obj.c1.l1[1].binary = b"yang" self.serialise_obj.c1.l1[1].boolean = True self.serialise_obj.c1.l1[1].enumeration = "one" self.serialise_obj.c1.l1[1].identityref = "idone" diff --git a/tests/serialise/xml-serialise/xml/obj.xml b/tests/serialise/xml-serialise/xml/obj.xml index 00723e9a..d37c0e4f 100644 --- a/tests/serialise/xml-serialise/xml/obj.xml +++ b/tests/serialise/xml-serialise/xml/obj.xml @@ -19,7 +19,7 @@ 16 chicken 16 - 010101 + eWFuZw== true one idone diff --git a/tests/union/run.py b/tests/union/run.py index 53edf7dc..880aef14 100755 --- a/tests/union/run.py +++ b/tests/union/run.py @@ -5,7 +5,6 @@ import unittest import six -from bitarray import bitarray from tests.base import PyangBindTestCase @@ -70,7 +69,7 @@ def test_default_value_of_int_typedef_within_union_typedef(self): self.assertEqual(self.instance.container.u8._default, 10) def test_leaf_list_with_union_of_unions_from_typedefs(self): - for value, valid in [(1, True), ("hello", True), (42.42, True), (True, True), (bitarray(10), False)]: + for value, valid in [(1, True), ("hello", True), (42.42, True), (True, True), (b"yang", False)]: with self.subTest(value=value, valid=valid): allowed = True try: diff --git a/tox.ini b/tox.ini index 63abb240..0e95ed39 100644 --- a/tox.ini +++ b/tox.ini @@ -22,9 +22,6 @@ testpaths = addopts = --disable-warnings --import-mode importlib - # (TODO): https://github.com/robshakir/pyangbind/issues/304 - --ignore=tests/binary # (TODO): https://github.com/robshakir/pyangbind/issues/305 --ignore=tests/integration/openconfig-bgp --ignore=tests/serialise/juniper-json-examples - \ No newline at end of file