Handle YANG binary type

As described in RFC7959, section 9.8.2, values of the ``binary``
built-in type are represented with the base64 encoding scheme.
Encode / decode from that scheme on serialise/deserialise, and
use native ``bytes`` type for Python representation.

docs/yang.md

@@ -52,7 +52,7 @@ PyangBind does not currently try and be feature complete against the YANG langua
 
  **Type**            | **Sub-Statement**   | **Supported Type**      | **Unit Tests**
  --------------------|--------------------|--------------------------|---------------
- **binary**          | -                   | [bitarray](https://github.com/ilanschnell/bitarray)           | tests/binary
+ **binary**          | -                   | [bytes](https://docs.python.org/3/library/stdtypes.html?#bytes)           | tests/binary
  -                   | length              | Supported           | tests/binary
  **bits**            | -                   | Not supported           | N/A
  -                   | position            | Not supported           | N/A

pyangbind/lib/serialise.py

@@ -28,6 +28,7 @@
 import json
 from collections import OrderedDict
 from decimal import Decimal
+import base64
 
 import six
 from enum import IntEnum
@@ -130,7 +131,7 @@ def default(self, obj):
         elif orig_yangt in ["string", "enumeration"]:
             return six.text_type(obj)
         elif orig_yangt in ["binary"]:
-            return obj.to01()
+            return six.text_type(base64.b64encode(obj), "ascii")
         elif orig_yangt in ["decimal64"]:
             return self.yangt_decimal(obj)
         elif orig_yangt in ["bool"]:
@@ -180,8 +181,8 @@ def map_pyangbind_type(self, map_val, original_yang_type, obj):
         elif map_val in ["pyangbind.lib.yangtypes.RestrictedPrecisionDecimal", "RestrictedPrecisionDecimal"]:
             # NOTE: this doesn't seem like it needs to be a special case?
             return self.yangt_decimal(obj)
-        elif map_val in ["bitarray.bitarray"]:
-            return obj.to01()
+        elif map_val in ["pyangbind.lib.yangtypes.YANGBinary", "YANGBinary"]:
+            return six.text_type(base64.b64encode(obj), "ascii")
         elif map_val in ["unicode"]:
             return six.text_type(obj)
         elif map_val in ["pyangbind.lib.yangtypes.YANGBool"]:

pyangbind/lib/yangtypes.py

@@ -21,6 +21,7 @@
 """
 from __future__ import unicode_literals
 
+import base64
 import collections
 from collections import abc
 import copy
@@ -29,7 +30,6 @@
 
 import regex
 import six
-from bitarray import bitarray
 
 # Words that could turn up in YANG definition files that are actually
 # reserved names in Python, such as being builtin types. This list is
@@ -274,9 +274,7 @@ def build_length_range_tuples(range, length=False, multiplier=1):
 
             def in_range_check(low_high_tuples, length=False):
                 def range_check(value):
-                    if length and isinstance(value, bitarray):
-                        value = value.length()
-                    elif length:
+                    if length:
                         value = len(value)
                     range_results = []
                     for check_tuple in low_high_tuples:
@@ -334,12 +332,7 @@ def in_dictionary_check(dictionary):
                         except Exception:
                             raise TypeError("must specify a numeric type for a range " + "argument")
                 elif rtype == "length":
-                    # When the type is a binary then the length is specified in
-                    # octets rather than bits, so we must specify the length to
-                    # be multiplied by 8.
                     multiplier = 1
-                    if base_type == bitarray:
-                        multiplier = 8
                     lengths = []
                     for range_spec in rarg:
                         lengths.append(build_length_range_tuples(range_spec, length=True, multiplier=multiplier))
@@ -1352,3 +1345,28 @@ def __str__(self):
             return str(self._get_ptr())
 
     return type(ReferencePathType(*args, **kwargs))
+
+
+class YANGBinary(bytes):
+    """
+    A custom binary class for using in YANG.
+    """
+
+    def __new__(self, *args, **kwargs):
+        value = b""
+        if args:
+            value = args[0]
+            if isinstance(value, str):
+                value = base64.b64decode(value)
+            elif isinstance(value, bytes):
+                value = value
+            else:
+                raise ValueError(f"invalid type for {value}: {type(value)}")
+
+        return bytes.__new__(self, value)
+
+    def __repr__(self):
+        return str(self)
+
+    def __str__(self, encoding="ascii", errors="replace"):
+        return str(self, encoding=encoding, errors=errors)

pyangbind/plugin/pybind.py

@@ -30,12 +30,11 @@
 from collections import OrderedDict
 
 import six
-from bitarray import bitarray
 from pyang import plugin, statements, util
 
 import pyangbind.helpers.misc as misc_help
 from pyangbind.helpers.identity import IdentityStore
-from pyangbind.lib.yangtypes import RestrictedClassType, YANGBool, safe_name
+from pyangbind.lib.yangtypes import RestrictedClassType, YANGBool, safe_name, YANGBinary
 
 # Python3 support
 if six.PY3:
@@ -100,7 +99,7 @@
         "quote_arg": True,
         "pytype": YANGBool,
     },
-    "binary": {"native_type": "bitarray", "base_type": True, "quote_arg": True, "pytype": bitarray},
+    "binary": {"native_type": "YANGBinary", "base_type": True, "quote_arg": True, "pytype": YANGBinary},
     "uint8": {
         "native_type": ("RestrictedClassType(base_type=int," + " restriction_dict={'range': ['0..255']}, int_size=8)"),
         "base_type": True,
@@ -317,13 +316,13 @@ def build_pybind(ctx, modules, fd):
         "YANGListType",
         "YANGDynClass",
         "ReferenceType",
+        "YANGBinary",
     ]
     for library in yangtypes_imports:
         ctx.pybind_common_hdr += "from pyangbind.lib.yangtypes import {}\n".format(library)
     ctx.pybind_common_hdr += "from pyangbind.lib.base import PybindBase\n"
     ctx.pybind_common_hdr += "from collections import OrderedDict\n"
     ctx.pybind_common_hdr += "from decimal import Decimal\n"
-    ctx.pybind_common_hdr += "from bitarray import bitarray\n"
     ctx.pybind_common_hdr += "import six\n"
 
     # Python3 support

requirements.txt

@@ -1,5 +1,4 @@
 pyang
-bitarray
 lxml
 regex
 six

tests/binary/binary.yang

@@ -21,7 +21,7 @@ module binary {
 
         leaf b2 {
             type binary;
-            default "0100";
+            default "eWFuZw=="; /* yang */
             description
                 "A test leaf with a default";
         }

tests/binary/run.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-from bitarray import bitarray
 import unittest
 
 from tests.base import PyangBindTestCase
@@ -19,8 +18,8 @@ def test_binary_leafs_exist(self):
                     hasattr(self.binary_obj.container, leaf), "Element did not exist in container (%s)" % leaf
                 )
 
-    def test_set_bitarray_from_different_datatypes(self):
-        for value in [("01110", True), ({"42": 42}, True), (-42, False), ("Arthur Dent", False)]:
+    def test_set_binary_from_different_datatypes(self):
+        for value in [(b"42", True), ({"42": 42}, False), (-42, False), ("Arthur Dent", False)]:
             with self.subTest(value=value):
                 allowed = True
                 try:
@@ -30,46 +29,46 @@ def test_set_bitarray_from_different_datatypes(self):
                 self.assertEqual(allowed, value[1], "Could incorrectly set b1 to %s" % value[0])
 
     def test_binary_leaf_default_value(self):
-        default = bitarray("0100")
+        default = b"yang"
         self.assertEqual(
             self.binary_obj.container.b2._default,
             default,
             "Default for leaf b2 was not set correctly (%s != %s)" % (self.binary_obj.container.b2._default, default),
         )
 
-    def test_binary_leaf_is_empty_bitarray_by_default(self):
-        empty = bitarray()
+    def test_binary_leaf_is_empty_by_default(self):
+        empty = b""
         self.assertEqual(
             self.binary_obj.container.b2,
             empty,
-            "Value of bitarray was not null when checking b2 (%s != %s)" % (self.binary_obj.container.b2, empty),
+            "Value of binary leaf was not null when checking b2 (%s != %s)" % (self.binary_obj.container.b2, empty),
         )
 
     def test_binary_leaf_is_not_changed_by_default(self):
         self.assertFalse(
             self.binary_obj.container.b2._changed(),
-            "Unset bitarray specified changed when was default (%s != False)"
+            "Unset binary leaf specified changed when was default (%s != False)"
             % self.binary_obj.container.b2._changed(),
         )
 
-    def test_set_bitarray_stores_value(self):
-        bits = bitarray("010")
+    def test_set_binary_stores_value(self):
+        bits = b"010"
         self.binary_obj.container.b2 = bits
         self.assertEqual(
             self.binary_obj.container.b2,
             bits,
-            "Bitarray not successfully set (%s != %s)" % (self.binary_obj.container.b2, bits),
+            "Binary leaf not successfully set (%s != %s)" % (self.binary_obj.container.b2, bits),
         )
 
-    def test_setting_bitarray_set_changed(self):
-        self.binary_obj.container.b2 = bitarray("010")
+    def test_setting_binary_set_changed(self):
+        self.binary_obj.container.b2 = b"010"
         self.assertTrue(
             self.binary_obj.container.b2._changed(),
-            "Bitarray value not flagged as changed (%s != True)" % self.binary_obj.container.b2._changed(),
+            "Binary leaf value not flagged as changed (%s != True)" % self.binary_obj.container.b2._changed(),
         )
 
-    def test_set_specific_length_bitarray(self):
-        for bits in [("0", False), ("1000000011110000", True), ("111111110000000011111111", False)]:
+    def test_set_specific_length_binary_leaf(self):
+        for bits in [(b"1", False), (b"12", True), (b"1234", False)]:
             with self.subTest(bits=bits):
                 allowed = True
                 try:
@@ -82,13 +81,8 @@ def test_set_specific_length_bitarray(self):
                     "limited length binary incorrectly set to %s (%s != %s)" % (bits[0], bits[1], allowed),
                 )
 
-    def test_set_bitarray_with_length_range(self):
-        for bits in [
-            ("0", False),
-            ("1111111100000000", True),
-            ("111111110000000011111111", True),
-            ("1111111100000000111111110000000011110000", False),
-        ]:
+    def test_set_binary_leaf_with_length_range(self):
+        for bits in [(b"1", False), (b"12", True), (b"1234", True), (b"12345", False)]:
             with self.subTest(bits=bits):
                 allowed = True
                 try:
@@ -101,25 +95,16 @@ def test_set_bitarray_with_length_range(self):
                     "Limited length binary with range incorrectly set to %s (%s != %s)" % (bits[0], bits[1], allowed),
                 )
 
-    def test_set_bitarray_with_complex_length(self):
+    def test_set_binary_leaf_with_complex_length(self):
         for bits in [
-            ("0", False),
-            ("1111000011110000", True),
-            ("111100001111000011110000", True),
-            ("1111000011110000111100001111000011110000", False),
-            ("111100001111000011110000111100001111000011110000", True),
-            ("111100001111000011110000111100001111000011110000" "11110000111100001111000011110000", True),
-            (
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "111100001111000011110000111100001111000011110000"
-                "1010101010101010",
-                False,
-            ),
+            (b"1", False),
+            (b"12", True),
+            (b"123", True),
+            (b"12345", False),
+            (b"123456", True),
+            (b"123456789_", True),
+            (b"123456789_123456789_123456789_123456789_12", True),
+            (b"123456789_123456789_123456789_123456789_123", False),
         ]:
             with self.subTest(bits=bits):
                 allowed = True

tests/serialise/ietf-json-deserialise/json/complete-obj.json

@@ -42,7 +42,7 @@
                 "one-leaf": "hi", 
                 "typedef-one": "test", 
                 "boolean": true, 
-                "binary": "010101", 
+                "binary": "eWFuZw==",
                 "union": "16", 
                 "k1": 1, 
                 "enumeration": "one", 

tests/serialise/ietf-json-deserialise/run.py

@@ -7,8 +7,6 @@
 from collections import OrderedDict
 from decimal import Decimal
 
-from bitarray import bitarray
-
 from pyangbind.lib.serialise import pybindJSONDecoder
 from tests.base import PyangBindTestCase
 
@@ -58,7 +56,7 @@ def test_all_the_types(self):
                         "one-leaf": "hi",
                         "typedef-one": "test",
                         "boolean": True,
-                        "binary": bitarray("010101"),
+                        "binary": b"yang",
                         "union": "16",
                         "identityref": "idone",
                         "enumeration": "one",

tests/serialise/ietf-json-serialise/json/obj.json

@@ -42,7 +42,7 @@
                 "one-leaf": "hi", 
                 "typedef-one": "test", 
                 "boolean": true, 
-                "binary": "010101", 
+                "binary": "eWFuZw==",
                 "union": "16", 
                 "k1": 1, 
                 "enumeration": "one", 

tests/serialise/ietf-json-serialise/run.py

@@ -6,7 +6,6 @@
 from decimal import Decimal
 
 import six
-from bitarray import bitarray
 
 from pyangbind.lib.serialise import pybindIETFJSONEncoder
 from pyangbind.lib.xpathhelper import YANGPathHelper
@@ -52,7 +51,7 @@ def test_serialise_full_container(self):
         self.serialise_obj.c1.t1.add(32)
         self.serialise_obj.c1.l1[1].leafref = 16
 
-        self.serialise_obj.c1.l1[1].binary = bitarray("010101")
+        self.serialise_obj.c1.l1[1].binary = b"yang"
         self.serialise_obj.c1.l1[1].boolean = True
         self.serialise_obj.c1.l1[1].enumeration = "one"
         self.serialise_obj.c1.l1[1].identityref = "idone"

tests/serialise/json-deserialise/json/alltypes.json

@@ -13,7 +13,7 @@
                 "one-leaf": "hi", 
                 "typedef-one": "test", 
                 "boolean": true, 
-                "binary": "010101", 
+                "binary": "eWFuZw==",
                 "union": "16", 
                 "k1": 1, 
                 "enumeration": "one", 

tests/serialise/json-deserialise/run.py

@@ -4,7 +4,6 @@
 import json
 import os.path
 import unittest
-from bitarray import bitarray
 from decimal import Decimal
 
 import pyangbind.lib.pybindJSON as pbJ
@@ -60,7 +59,7 @@ def test_all_the_types(self):
                         "one-leaf": "hi",
                         "typedef-one": "test",
                         "boolean": True,
-                        "binary": bitarray("010101"),
+                        "binary": b"yang",
                         "union": "16",
                         "identityref": "idone",
                         "enumeration": "one",

tests/serialise/json-serialise/json/expected-output.json

@@ -51,7 +51,7 @@
                 "one-leaf": "hi", 
                 "typedef-one": "test", 
                 "boolean": true, 
-                "binary": "010101", 
+                "binary": "eWFuZw==",
                 "union": "16", 
                 "k1": 1, 
                 "enumeration": "one", 

tests/serialise/json-serialise/run.py

@@ -6,7 +6,6 @@
 from decimal import Decimal
 
 import six
-from bitarray import bitarray
 
 from pyangbind.lib.pybindJSON import dumps
 from pyangbind.lib.xpathhelper import YANGPathHelper
@@ -48,7 +47,7 @@ def test_full_serialise(self):
         self.serialise_obj.c1.t1.add(32)
         self.serialise_obj.c1.l1[1].leafref = 16
 
-        self.serialise_obj.c1.l1[1].binary = bitarray("010101")
+        self.serialise_obj.c1.l1[1].binary = b"yang"
         self.serialise_obj.c1.l1[1].boolean = True
         self.serialise_obj.c1.l1[1].enumeration = "one"
         self.serialise_obj.c1.l1[1].identityref = "idone"