Merge pull request #2 from andrewl33/add-vecbytes

Add support for nested byte vectors
aptos-labs · Sep 7, 2022 · 1c88912 · 1c88912
2 parents 839aed6 + ea6ab19
commit 1c88912
Show file tree

Hide file tree

Showing 18 changed files with 176 additions and 0 deletions.
diff --git a/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs b/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs
@@ -65,6 +65,22 @@ public string deserialize_str()
             return utf8.GetString(content);
         }
 
+        public ValueArray<ValueArray<byte>> deserialize_vec_bytes() {
+            long len = deserialize_len();
+            ValueArray<byte>[] content = new ValueArray<byte>[len];
+
+            if (len < 0 || len > int.MaxValue)
+            {
+                throw new DeserializationException("Incorrect length value for C# array");
+            }
+
+            for (long i = 0; i < len; i++) {
+                content[i] = deserialize_bytes();
+            }
+
+            return new ValueArray<ValueArray<byte>>(content);
+        }
+
         public ValueArray<byte> deserialize_bytes()
         {
             long len = deserialize_len();

diff --git a/serde-generate/runtime/csharp/Serde/BinarySerializer.cs b/serde-generate/runtime/csharp/Serde/BinarySerializer.cs
@@ -64,6 +64,13 @@ public void decrease_container_depth()
 
         public void serialize_str(string value) => serialize_bytes(new ValueArray<byte>(utf8.GetBytes(value)));
 
+        public void serialize_vec_bytes(ValueArray<ValueArray<byte>> value) {
+            serialize_len(value.Count);
+            foreach (ValueArray<byte> v in value) {
+                serialize_bytes(v);
+            }
+        }
+
         public void serialize_bytes(ValueArray<byte> value)
         {
             serialize_len(value.Count);

diff --git a/serde-generate/runtime/csharp/Serde/IDeserializer.cs b/serde-generate/runtime/csharp/Serde/IDeserializer.cs
@@ -9,6 +9,8 @@ public interface IDeserializer
     {
         string deserialize_str();
 
+        ValueArray<ValueArray<byte>> deserialize_vec_bytes();
+
         ValueArray<byte> deserialize_bytes();
 
         bool deserialize_bool();

diff --git a/serde-generate/runtime/csharp/Serde/ISerializer.cs b/serde-generate/runtime/csharp/Serde/ISerializer.cs
@@ -9,6 +9,8 @@ public interface ISerializer
     {
         void serialize_str(string value);
 
+        void serialize_vec_bytes(ValueArray<ValueArray<byte>> value);
+
         void serialize_bytes(ValueArray<byte> value);
 
         void serialize_bool(bool value);

diff --git a/serde-generate/runtime/golang/bcs/bcs_test.go b/serde-generate/runtime/golang/bcs/bcs_test.go
@@ -50,6 +50,47 @@ func TestSerializeDeserializeBytes(t *testing.T) {
 	})
 }
 
+func TestSerializeDeserializeVecBytes(t *testing.T) {
+	cases := []struct {
+		target   [][]byte
+		expected []byte
+	}{
+		{
+			target:   [][]byte{{1, 2, 38}, {0, 1}, {0}},
+			expected: []byte{3, 3, 1, 2, 38, 2, 0, 1, 1, 0},
+		},
+		{
+			target:   [][]byte{{1, 2, 38}, {0, 1}, {}},
+			expected: []byte{3, 3, 1, 2, 38, 2, 0, 1, 0},
+		},
+		{
+			target:   [][]byte{},
+			expected: []byte{0},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%#v", tc.target), func(t *testing.T) {
+			s := bcs.NewSerializer()
+			d := bcs.NewDeserializer(tc.expected)
+
+			err := s.SerializeVecBytes(tc.target)
+			require.NoError(t, err)
+
+			deserialized, err := d.DeserializeVecBytes()
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expected, s.GetBytes())
+			assert.Equal(t, tc.target, deserialized)
+		})
+	}
+	t.Run("deserialize error: EOF", func(t *testing.T) {
+		d := bcs.NewDeserializer([]byte{})
+		_, err := d.DeserializeVecBytes()
+		require.EqualError(t, err, "EOF")
+	})
+}
+
 func TestSerializeDeserializeStr(t *testing.T) {
 	cases := []struct {
 		target   string

diff --git a/serde-generate/runtime/golang/bcs/deserializer.go b/serde-generate/runtime/golang/bcs/deserializer.go
@@ -37,6 +37,10 @@ func (d *deserializer) DeserializeF64() (float64, error) {
 	return 0, errors.New("unimplemented")
 }
 
+func (d *deserializer) DeserializeVecBytes() ([][]byte, error) {
+	return d.BinaryDeserializer.DeserializeVecBytes(d.DeserializeLen)
+}
+
 func (d *deserializer) DeserializeBytes() ([]byte, error) {
 	return d.BinaryDeserializer.DeserializeBytes(d.DeserializeLen)
 }

diff --git a/serde-generate/runtime/golang/bcs/serializer.go b/serde-generate/runtime/golang/bcs/serializer.go
@@ -34,6 +34,10 @@ func (s *serializer) SerializeStr(value string) error {
 	return s.BinarySerializer.SerializeStr(value, s.SerializeLen)
 }
 
+func (s *serializer) SerializeVecBytes(value [][]byte) error {
+	return s.BinarySerializer.SerializeVecBytes(value, s.SerializeLen)
+}
+
 func (s *serializer) SerializeBytes(value []byte) error {
 	return s.BinarySerializer.SerializeBytes(value, s.SerializeLen)
 }

diff --git a/serde-generate/runtime/golang/bincode/deserializer.go b/serde-generate/runtime/golang/bincode/deserializer.go
@@ -32,6 +32,10 @@ func (d *deserializer) DeserializeF64() (float64, error) {
 	return math.Float64frombits(ret), err
 }
 
+func (d *deserializer) DeserializeVecBytes() ([][]byte, error) {
+	return d.BinaryDeserializer.DeserializeVecBytes(d.DeserializeLen)
+}
+
 func (d *deserializer) DeserializeBytes() ([]byte, error) {
 	return d.BinaryDeserializer.DeserializeBytes(d.DeserializeLen)
 }

diff --git a/serde-generate/runtime/golang/bincode/serializer.go b/serde-generate/runtime/golang/bincode/serializer.go
@@ -30,6 +30,10 @@ func (s *serializer) SerializeStr(value string) error {
 	return s.BinarySerializer.SerializeStr(value, s.SerializeLen)
 }
 
+func (s *serializer) SerializeVecBytes(value [][]byte) error {
+	return s.BinarySerializer.SerializeVecBytes(value, s.SerializeLen)
+}
+
 func (s *serializer) SerializeBytes(value []byte) error {
 	return s.BinarySerializer.SerializeBytes(value, s.SerializeLen)
 }

diff --git a/serde-generate/runtime/golang/serde/binary_deserializer.go b/serde-generate/runtime/golang/serde/binary_deserializer.go
@@ -38,6 +38,23 @@ func (d *BinaryDeserializer) DecreaseContainerDepth() {
 	d.containerDepthBudget += 1
 }
 
+func (d *BinaryDeserializer) DeserializeVecBytes(deserializeLen func() (uint64, error)) ([][]byte, error) {
+	len, err := deserializeLen()
+	if err != nil {
+		return nil, err
+	}
+	ret := make([][]byte, len)
+	for i := 0; uint64(i) < len; i++ {
+		bytes, err := d.DeserializeBytes(deserializeLen)
+		if err != nil {
+			return nil, err
+		}
+		ret[i] = bytes
+	}
+
+	return ret, err
+}
+
 // `deserializeLen` to be provided by the extending struct.
 func (d *BinaryDeserializer) DeserializeBytes(deserializeLen func() (uint64, error)) ([]byte, error) {
 	len, err := deserializeLen()

diff --git a/serde-generate/runtime/golang/serde/binary_serializer.go b/serde-generate/runtime/golang/serde/binary_serializer.go
@@ -33,6 +33,14 @@ func (d *BinarySerializer) DecreaseContainerDepth() {
 	d.containerDepthBudget += 1
 }
 
+func (s *BinarySerializer) SerializeVecBytes(value [][]byte, serializeLen func(uint64) error) error {
+	serializeLen(uint64(len(value)))
+	for _, bytes := range value {
+		s.SerializeBytes(bytes, serializeLen)
+	}
+	return nil
+}
+
 // `serializeLen` to be provided by the extending struct.
 func (s *BinarySerializer) SerializeBytes(value []byte, serializeLen func(uint64) error) error {
 	serializeLen(uint64(len(value)))

diff --git a/serde-generate/runtime/golang/serde/interfaces.go b/serde-generate/runtime/golang/serde/interfaces.go
@@ -6,6 +6,8 @@ package serde
 type Serializer interface {
 	SerializeStr(value string) error
 
+	SerializeVecBytes(value [][]byte) error
+
 	SerializeBytes(value []byte) error
 
 	SerializeBool(value bool) error
@@ -58,6 +60,8 @@ type Serializer interface {
 type Deserializer interface {
 	DeserializeStr() (string, error)
 
+	DeserializeVecBytes() ([][]byte, error)
+
 	DeserializeBytes() ([]byte, error)
 
 	DeserializeBool() (bool, error)

diff --git a/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java b/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java
@@ -9,6 +9,8 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.charset.CharacterCodingException;
 import java.math.BigInteger;
+import java.util.List;
+import java.util.ArrayList;
 
 public abstract class BinaryDeserializer implements Deserializer {
     protected ByteBuffer input;
@@ -47,6 +49,19 @@ public String deserialize_str() throws DeserializationError {
         return new String(content);
     }
 
+    public List<Bytes> deserialize_vec_bytes() throws DeserializationError {
+        long len = deserialize_len();
+        if (len < 0 || len > Integer.MAX_VALUE) {
+            throw new DeserializationError("Incorrect length value for Java array");
+        }
+        List<Bytes> content = new ArrayList<Bytes>();
+        for (int i = 0; i < len; i++) {
+            content.add(deserialize_bytes());
+        }
+
+        return content;
+    }
+
     public Bytes deserialize_bytes() throws DeserializationError {
         long len = deserialize_len();
         if (len < 0 || len > Integer.MAX_VALUE) {

diff --git a/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java b/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java
@@ -4,6 +4,7 @@
 package com.novi.serde;
 
 import java.math.BigInteger;
+import java.util.List;
 
 public abstract class BinarySerializer implements Serializer {
     protected MyByteArrayOutputStream output;
@@ -29,6 +30,13 @@ public void serialize_str(String value) throws SerializationError {
         serialize_bytes(new Bytes(value.getBytes()));
     }
 
+    public void serialize_vec_bytes(List<Bytes> value) throws SerializationError {
+        serialize_len(value.size());
+        for (Bytes bytes : value) {
+            serialize_bytes(bytes);
+        }
+    }
+
     public void serialize_bytes(Bytes value) throws SerializationError {
         byte[] content = value.content();
         serialize_len(content.length);

diff --git a/serde-generate/runtime/java/com/novi/serde/Deserializer.java b/serde-generate/runtime/java/com/novi/serde/Deserializer.java
@@ -4,10 +4,13 @@
 package com.novi.serde;
 
 import java.math.BigInteger;
+import java.util.List;
 
 public interface Deserializer {
     String deserialize_str() throws DeserializationError;
 
+    List<Bytes> deserialize_vec_bytes() throws DeserializationError;
+
     Bytes deserialize_bytes() throws DeserializationError;
 
     Boolean deserialize_bool() throws DeserializationError;

diff --git a/serde-generate/runtime/java/com/novi/serde/Serializer.java b/serde-generate/runtime/java/com/novi/serde/Serializer.java
@@ -4,10 +4,13 @@
 package com.novi.serde;
 
 import java.math.BigInteger;
+import java.util.List;
 
 public interface Serializer {
     void serialize_str(String value) throws SerializationError;
 
+    void serialize_vec_bytes(List<Bytes> value) throws SerializationError;
+
     void serialize_bytes(Bytes value) throws SerializationError;
 
     void serialize_bool(Boolean value) throws SerializationError;

diff --git a/serde-generate/runtime/python/bcs/test_bcs.py b/serde-generate/runtime/python/bcs/test_bcs.py
@@ -141,6 +141,28 @@ def test_serialize_bytes(self):
 
         self.assertEqual(bcs.deserialize(b"\x00", bytes), (b"", b""))
 
+    def test_serialize_vec_bytes(self):
+        a = []
+        b = [b"\x00\x00", b"\x01\x01"]
+        c = [b"\x00" * 128, b"\x01" * 127, b"\x02" * 3, b""]
+
+        aser = bcs.serialize(a, typing.Sequence[bytes])
+        bser = bcs.serialize(b, typing.Sequence[bytes])
+        cser = bcs.serialize(c, typing.Sequence[bytes])
+
+        self.assertEqual(aser, b"\x00")
+        self.assertEqual(bser, b"\x02\x02\x00\x00\x02\x01\x01")
+        self.assertEqual(cser,
+                b"\x04\x80\x01" + b"\x00" * 128 +
+                b"\x7F" + b"\x01" * 127 +
+                b"\x03" + b"\x02" * 3 +
+                b"\x00"
+        )
+
+        self.assertEqual(bcs.deserialize(aser, typing.Sequence[bytes]), (a, b""))
+        self.assertEqual(bcs.deserialize(bser, typing.Sequence[bytes]), (b, b""))
+        self.assertEqual(bcs.deserialize(cser, typing.Sequence[bytes]), (c, b""))
+
     def test_serialize_tuple(self):
         T = typing.Tuple[st.uint8, st.uint16]
         self.assertEqual(bcs.serialize((0, 1), T), b"\x00\x01\x00")

diff --git a/serde-generate/runtime/python/serde_binary/__init__.py b/serde-generate/runtime/python/serde_binary/__init__.py
@@ -47,8 +47,15 @@ def __post_init__(self):
             st.char: self.serialize_char,
             str: self.serialize_str,
             bytes: self.serialize_bytes,
+            typing.Sequence[bytes]: self.serialize_vec_bytes,
         }
 
+    def serialize_vec_bytes(self, value: typing.Sequence[bytes]):
+        self.serialize_len(len(value))
+        for byte_vec in value:
+            self.serialize_bytes(byte_vec)
+
+
     def serialize_bytes(self, value: bytes):
         self.serialize_len(len(value))
         self.output.write(value)
@@ -227,6 +234,7 @@ def __post_init__(self):
             st.char: self.deserialize_char,
             str: self.deserialize_str,
             bytes: self.deserialize_bytes,
+            typing.Sequence[bytes]: self.deserialize_vec_bytes,
         }
 
     def read(self, length: int) -> bytes:
@@ -235,6 +243,10 @@ def read(self, length: int) -> bytes:
             raise st.DeserializationError("Input is too short")
         return value
 
+    def deserialize_vec_bytes(self) -> typing.Sequence[bytes]:
+        length = self.deserialize_len()
+        return [self.deserialize_bytes() for _ in range(length)]
+
     def deserialize_bytes(self) -> bytes:
         length = self.deserialize_len()
         return self.read(length)