Improve WriteString perf with SIMD

protocolbuffers · Dec 16, 2020 · 9189e56 · 9189e56
1 parent 5f91454
commit 9189e56
Show file tree

Hide file tree

Showing 9 changed files with 137 additions and 34 deletions.
diff --git a/csharp/src/Google.Protobuf.Test/Buffers/ArrayBufferWriter.cs b/csharp/src/Google.Protobuf.Test/Buffers/ArrayBufferWriter.cs
@@ -42,18 +42,18 @@ namespace Google.Protobuf.Buffers
     /// ArrayBufferWriter is originally from corefx, and has been contributed to Protobuf
     /// https://github.com/dotnet/runtime/blob/071da4c41aa808c949a773b92dca6f88de9d11f3/src/libraries/Common/src/System/Buffers/ArrayBufferWriter.cs
     /// </summary>
-    internal sealed class ArrayBufferWriter<T> : IBufferWriter<T>
+    internal sealed class TestArrayBufferWriter<T> : IBufferWriter<T>
     {
         private T[] _buffer;
         private int _index;
 
         private const int DefaultInitialBufferSize = 256;
 
         /// <summary>
-        /// Creates an instance of an <see cref="ArrayBufferWriter{T}"/>, in which data can be written to,
+        /// Creates an instance of an <see cref="TestArrayBufferWriter{T}"/>, in which data can be written to,
         /// with the default initial capacity.
         /// </summary>
-        public ArrayBufferWriter()
+        public TestArrayBufferWriter()
         {
             _buffer = new T[0];
             _index = 0;
@@ -66,14 +66,14 @@ public ArrayBufferWriter()
         public int? MaxGrowBy { get; set; }
 
         /// <summary>
-        /// Creates an instance of an <see cref="ArrayBufferWriter{T}"/>, in which data can be written to,
+        /// Creates an instance of an <see cref="TestArrayBufferWriter{T}"/>, in which data can be written to,
         /// with an initial capacity specified.
         /// </summary>
         /// <param name="initialCapacity">The minimum capacity with which to initialize the underlying buffer.</param>
         /// <exception cref="ArgumentException">
         /// Thrown when <paramref name="initialCapacity"/> is not positive (i.e. less than or equal to 0).
         /// </exception>
-        public ArrayBufferWriter(int initialCapacity)
+        public TestArrayBufferWriter(int initialCapacity)
         {
             if (initialCapacity <= 0)
                 throw new ArgumentException(nameof(initialCapacity));
@@ -111,7 +111,7 @@ public ArrayBufferWriter(int initialCapacity)
         /// Clears the data written to the underlying buffer.
         /// </summary>
         /// <remarks>
-        /// You must clear the <see cref="ArrayBufferWriter{T}"/> before trying to re-use it.
+        /// You must clear the <see cref="TestArrayBufferWriter{T}"/> before trying to re-use it.
         /// </remarks>
         public void Clear()
         {

diff --git a/csharp/src/Google.Protobuf.Test/CodedOutputStreamTest.cs b/csharp/src/Google.Protobuf.Test/CodedOutputStreamTest.cs
@@ -57,7 +57,7 @@ private static void AssertWriteVarint(byte[] data, ulong value)
                 Assert.AreEqual(data, rawOutput.ToArray());
 
                 // IBufferWriter
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteUInt32((uint) value);
                 ctx.Flush();
@@ -76,7 +76,7 @@ private static void AssertWriteVarint(byte[] data, ulong value)
                 Assert.AreEqual(data, rawOutput.ToArray());
 
                 // IBufferWriter
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteUInt64(value);
                 ctx.Flush();
@@ -99,7 +99,7 @@ private static void AssertWriteVarint(byte[] data, ulong value)
                     output.Flush();
                     Assert.AreEqual(data, rawOutput.ToArray());
 
-                    var bufferWriter = new ArrayBufferWriter<byte>();
+                    var bufferWriter = new TestArrayBufferWriter<byte>();
                     bufferWriter.MaxGrowBy = bufferSize;
                     WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                     ctx.WriteUInt32((uint) value);
@@ -114,7 +114,7 @@ private static void AssertWriteVarint(byte[] data, ulong value)
                     output.Flush();
                     Assert.AreEqual(data, rawOutput.ToArray());
 
-                    var bufferWriter = new ArrayBufferWriter<byte>();
+                    var bufferWriter = new TestArrayBufferWriter<byte>();
                     bufferWriter.MaxGrowBy = bufferSize;
                     WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                     ctx.WriteUInt64(value);
@@ -173,7 +173,7 @@ private static void AssertWriteLittleEndian32(byte[] data, uint value)
                 output.Flush();
                 Assert.AreEqual(data, rawOutput.ToArray());
 
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteFixed32(value);
                 ctx.Flush();
@@ -189,7 +189,7 @@ private static void AssertWriteLittleEndian32(byte[] data, uint value)
                 output.Flush();
                 Assert.AreEqual(data, rawOutput.ToArray());
 
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 bufferWriter.MaxGrowBy = bufferSize;
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteFixed32(value);
@@ -211,7 +211,7 @@ private static void AssertWriteLittleEndian64(byte[] data, ulong value)
                 output.Flush();
                 Assert.AreEqual(data, rawOutput.ToArray());
 
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteFixed64(value);
                 ctx.Flush();
@@ -227,7 +227,7 @@ private static void AssertWriteLittleEndian64(byte[] data, ulong value)
                 output.Flush();
                 Assert.AreEqual(data, rawOutput.ToArray());
 
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 bufferWriter.MaxGrowBy = blockSize;
                 WriteContext.Initialize(bufferWriter, out WriteContext ctx);
                 ctx.WriteFixed64(value);
@@ -269,7 +269,7 @@ public void WriteWholeMessage_VaryingBlockSizes()
                 output.Flush();
                 Assert.AreEqual(rawBytes, rawOutput.ToArray());
 
-                var bufferWriter = new ArrayBufferWriter<byte>();
+                var bufferWriter = new TestArrayBufferWriter<byte>();
                 bufferWriter.MaxGrowBy = blockSize;
                 message.WriteTo(bufferWriter);
                 Assert.AreEqual(rawBytes, bufferWriter.WrittenSpan.ToArray()); 
@@ -291,7 +291,7 @@ public void WriteContext_WritesWithFlushes()
             output.Flush();
             byte[] expectedBytes2 = expectedOutput.ToArray();
 
-            var bufferWriter = new ArrayBufferWriter<byte>();
+            var bufferWriter = new TestArrayBufferWriter<byte>();
             WriteContext.Initialize(bufferWriter, out WriteContext ctx);
             ctx.WriteMessage(message);
             ctx.Flush();

diff --git a/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj b/csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net451;netcoreapp2.1</TargetFrameworks>
+    <TargetFrameworks>net451;netcoreapp2.1;net50</TargetFrameworks>
     <AssemblyOriginatorKeyFile>../../keys/Google.Protobuf.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
     <IsPackable>False</IsPackable>

diff --git a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs
@@ -551,9 +551,11 @@ public void NumberToDouble_Valid(string jsonValue, double expectedParsedValue)
         }
 
         [Test]
+#if !NET5_0
         [TestCase("1.7977e308")]
         [TestCase("-1.7977e308")]
         [TestCase("1e309")]
+#endif
         [TestCase("1,0")]
         [TestCase("1.0.0")]
         [TestCase("+1")]

diff --git a/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs b/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs
@@ -199,8 +199,10 @@ public void NumberValue(string json, double expectedValue)
         [TestCase("1e-")]
         [TestCase("--")]
         [TestCase("--1")]
+#if !NET5_0
         [TestCase("-1.7977e308")]
         [TestCase("1.7977e308")]
+#endif
         public void InvalidNumberValue(string json)
         {
             AssertThrowsAfter(json);

diff --git a/csharp/src/Google.Protobuf.Test/LegacyGeneratedCodeTest.cs b/csharp/src/Google.Protobuf.Test/LegacyGeneratedCodeTest.cs
@@ -141,7 +141,7 @@ public void LegacyGeneratedCodeThrowsWithIBufferWriter()
             };
             var exception = Assert.Throws<InvalidProtocolBufferException>(() =>
             {
-                WriteContext.Initialize(new ArrayBufferWriter<byte>(), out WriteContext writeCtx);
+                WriteContext.Initialize(new TestArrayBufferWriter<byte>(), out WriteContext writeCtx);
                 ((IBufferMessage)message).InternalWriteTo(ref writeCtx);
             });
             Assert.AreEqual($"Message {typeof(LegacyGeneratedCodeMessageA).Name} doesn't provide the generated method that enables WriteContext-based serialization. You might need to regenerate the generated protobuf code.", exception.Message);

diff --git a/csharp/src/Google.Protobuf.Test/MessageParsingHelpers.cs b/csharp/src/Google.Protobuf.Test/MessageParsingHelpers.cs
@@ -83,7 +83,7 @@ public static void AssertReadingMessage(MessageParser parser, byte[] bytes, Acti
             var bytes = message.ToByteArray();
 
             // also serialize using IBufferWriter and check it leads to the same data
-            var bufferWriter = new ArrayBufferWriter<byte>();
+            var bufferWriter = new TestArrayBufferWriter<byte>();
             message.WriteTo(bufferWriter);
             Assert.AreEqual(bytes, bufferWriter.WrittenSpan.ToArray(), "Both serialization approaches need to result in the same data.");
 
@@ -112,7 +112,7 @@ public static void AssertWritingMessage(IMessage message)
             Assert.AreEqual(message.CalculateSize(), bytes.Length);
 
             // serialize using IBufferWriter and check it leads to the same output
-            var bufferWriter = new ArrayBufferWriter<byte>();
+            var bufferWriter = new TestArrayBufferWriter<byte>();
             message.WriteTo(bufferWriter);
             Assert.AreEqual(bytes, bufferWriter.WrittenSpan.ToArray());
 
@@ -124,7 +124,7 @@ public static void AssertWritingMessage(IMessage message)
             // test for different IBufferWriter.GetSpan() segment sizes
             for (int blockSize = 1; blockSize < 256; blockSize *= 2)
             {
-                var segmentedBufferWriter = new ArrayBufferWriter<byte>();
+                var segmentedBufferWriter = new TestArrayBufferWriter<byte>();
                 segmentedBufferWriter.MaxGrowBy = blockSize;
                 message.WriteTo(segmentedBufferWriter);
                 Assert.AreEqual(bytes, segmentedBufferWriter.WrittenSpan.ToArray());

diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description>
@@ -8,7 +8,7 @@
     <!-- C# 7.2 is required for Span/BufferWriter/ReadOnlySequence -->
     <LangVersion>7.2</LangVersion>
     <Authors>Google Inc.</Authors>
-    <TargetFrameworks>netstandard1.1;netstandard2.0;net45</TargetFrameworks>
+    <TargetFrameworks>netstandard1.1;netstandard2.0;net45;net50</TargetFrameworks>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <AssemblyOriginatorKeyFile>../../keys/Google.Protobuf.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
@@ -27,15 +27,23 @@
     <DefineConstants>$(DefineConstants);GOOGLE_PROTOBUF_SUPPORT_FAST_STRING</DefineConstants>
   </PropertyGroup>
 
+  <PropertyGroup Condition=" '$(TargetFramework)' == 'net50' ">
+    <DefineConstants>$(DefineConstants);GOOGLE_PROTOBUF_SUPPORT_FAST_STRING;GOOGLE_PROTOBUF_SIMD</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
-    <PackageReference Include="System.Memory" Version="4.5.3"/>
     <PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="All" Version="1.0.0"/>
     <!-- Needed for the net45 build to work on Unix. See https://github.com/dotnet/designs/pull/33 -->
     <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" PrivateAssets="All" Version="1.0.0"/>
   </ItemGroup>
 
-  <!-- Needed for netcoreapp2.1 to work correctly. .NET is not able to load the assembly without this -->
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net45' OR '$(TargetFramework)' == 'netstandard1.1' ">
+    <PackageReference Include="System.Memory" Version="4.5.3"/>
+  </ItemGroup>
+
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
+    <PackageReference Include="System.Memory" Version="4.5.3"/>
+    <!-- Needed for netcoreapp2.1 to work correctly. .NET is not able to load the assembly without this -->
     <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.5.2"/>
   </ItemGroup>