Skip to content

Commit

Permalink
Add DiscardUnknownFields support for C#
Browse files Browse the repository at this point in the history 
By default, unknown fields are preserved when parsing. To discard
them, use a parser configured to do so:

var parser = MyMessage.Parser.WithDiscardUnknownFields(true);
  • Loading branch information
@jskeet
jskeet committed Jan 15, 2018
1 parent 9f80df0 commit 47b7d2c
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 56 deletions.
48 changes: 48 additions & 0 deletions csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#endregion

using System;
using System.IO;
using Google.Protobuf.TestProtos;
using NUnit.Framework;

Expand Down Expand Up @@ -124,5 +125,52 @@ public void TestClone()
Assert.AreEqual(message.CalculateSize(), otherEmptyMessage.CalculateSize());
Assert.AreEqual(message.ToByteArray(), otherEmptyMessage.ToByteArray());
}

[Test]
public void TestDiscardUnknownFields()
{
var message = SampleMessages.CreateFullTestAllTypes();
var goldenEmptyMessage = new TestEmptyMessage();
byte[] data = message.ToByteArray();
int fullSize = message.CalculateSize();

Action<IMessage> assertEmpty = msg =>
{
Assert.AreEqual(0, msg.CalculateSize());
Assert.AreEqual(goldenEmptyMessage, msg);
};

Action<IMessage> assertFull = msg => Assert.AreEqual(fullSize, msg.CalculateSize());

// Test the behavior of the parsers with and without discarding, both generic and non-generic.
MessageParser<TestEmptyMessage> retainingParser1 = TestEmptyMessage.Parser;
MessageParser retainingParser2 = retainingParser1;
MessageParser<TestEmptyMessage> discardingParser1 = retainingParser1.WithDiscardUnknownFields(true);
MessageParser discardingParser2 = retainingParser2.WithDiscardUnknownFields(true);

// Test parse from byte[]
assertFull(retainingParser1.ParseFrom(data));
assertFull(retainingParser2.ParseFrom(data));
assertEmpty(discardingParser1.ParseFrom(data));
assertEmpty(discardingParser2.ParseFrom(data));

// Test parse from byte[] with offset
assertFull(retainingParser1.ParseFrom(data, 0, data.Length));
assertFull(retainingParser2.ParseFrom(data, 0, data.Length));
assertEmpty(discardingParser1.ParseFrom(data, 0, data.Length));
assertEmpty(discardingParser2.ParseFrom(data, 0, data.Length));

// Test parse from CodedInputStream
assertFull(retainingParser1.ParseFrom(new CodedInputStream(data)));
assertFull(retainingParser2.ParseFrom(new CodedInputStream(data)));
assertEmpty(discardingParser1.ParseFrom(new CodedInputStream(data)));
assertEmpty(discardingParser2.ParseFrom(new CodedInputStream(data)));

// Test parse from Stream
assertFull(retainingParser1.ParseFrom(new MemoryStream(data)));
assertFull(retainingParser2.ParseFrom(new MemoryStream(data)));
assertEmpty(discardingParser1.ParseFrom(new MemoryStream(data)));
assertEmpty(discardingParser2.ParseFrom(new MemoryStream(data)));
}
}
}
5 changes: 5 additions & 0 deletions csharp/src/Google.Protobuf/CodedInputStream.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,11 @@ public long Position
/// </value>
public int RecursionLimit { get { return recursionLimit; } }

/// <summary>
/// Internal-only property; when set to true, unknown fields will be discarded while parsing.
/// </summary>
internal bool DiscardUnknownFields { get; set; }

/// <summary>
/// Disposes of this instance, potentially closing any underlying stream.
/// </summary>
Expand Down
102 changes: 61 additions & 41 deletions csharp/src/Google.Protobuf/MessageExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,8 @@ public static class MessageExtensions
/// </summary>
/// <param name="message">The message to merge the data into.</param>
/// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
public static void MergeFrom(this IMessage message, byte[] data)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = new CodedInputStream(data);
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}
public static void MergeFrom(this IMessage message, byte[] data) =>
MergeFrom(message, data, false);

/// <summary>
/// Merges data from the given byte array slice into an existing message.
Expand All @@ -60,42 +54,24 @@ public static void MergeFrom(this IMessage message, byte[] data)
/// <param name="data">The data containing the slice to merge, which must be protobuf-encoded binary data.</param>
/// <param name="offset">The offset of the slice to merge.</param>
/// <param name="length">The length of the slice to merge.</param>
public static void MergeFrom(this IMessage message, byte[] data, int offset, int length)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = new CodedInputStream(data, offset, length);
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}
public static void MergeFrom(this IMessage message, byte[] data, int offset, int length) =>
MergeFrom(message, data, offset, length, false);

/// <summary>
/// Merges data from the given byte string into an existing message.
/// </summary>
/// <param name="message">The message to merge the data into.</param>
/// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
public static void MergeFrom(this IMessage message, ByteString data)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = data.CreateCodedInput();
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}
public static void MergeFrom(this IMessage message, ByteString data) =>
MergeFrom(message, data, false);

/// <summary>
/// Merges data from the given stream into an existing message.
/// </summary>
/// <param name="message">The message to merge the data into.</param>
/// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
public static void MergeFrom(this IMessage message, Stream input)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(input, "input");
CodedInputStream codedInput = new CodedInputStream(input);
message.MergeFrom(codedInput);
codedInput.CheckReadEndOfStreamTag();
}
public static void MergeFrom(this IMessage message, Stream input) =>
MergeFrom(message, input, false);

/// <summary>
/// Merges length-delimited data from the given stream into an existing message.
Expand All @@ -106,14 +82,8 @@ public static void MergeFrom(this IMessage message, Stream input)
/// </remarks>
/// <param name="message">The message to merge the data into.</param>
/// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
public static void MergeDelimitedFrom(this IMessage message, Stream input)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(input, "input");
int size = (int) CodedInputStream.ReadRawVarint32(input);
Stream limitedStream = new LimitedInputStream(input, size);
message.MergeFrom(limitedStream);
}
public static void MergeDelimitedFrom(this IMessage message, Stream input) =>
MergeDelimitedFrom(message, input, false);

/// <summary>
/// Converts the given message into a byte array in protobuf encoding.
Expand Down Expand Up @@ -168,6 +138,56 @@ public static ByteString ToByteString(this IMessage message)
{
ProtoPreconditions.CheckNotNull(message, "message");
return ByteString.AttachBytes(message.ToByteArray());
}
}

// Implementations allowing unknown fields to be discarded.
internal static void MergeFrom(this IMessage message, byte[] data, bool discardUnknownFields)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = new CodedInputStream(data);
input.DiscardUnknownFields = discardUnknownFields;
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}

internal static void MergeFrom(this IMessage message, byte[] data, int offset, int length, bool discardUnknownFields)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = new CodedInputStream(data, offset, length);
input.DiscardUnknownFields = discardUnknownFields;
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}

internal static void MergeFrom(this IMessage message, ByteString data, bool discardUnknownFields)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(data, "data");
CodedInputStream input = data.CreateCodedInput();
input.DiscardUnknownFields = discardUnknownFields;
message.MergeFrom(input);
input.CheckReadEndOfStreamTag();
}

internal static void MergeFrom(this IMessage message, Stream input, bool discardUnknownFields)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(input, "input");
CodedInputStream codedInput = new CodedInputStream(input);
codedInput.DiscardUnknownFields = discardUnknownFields;
message.MergeFrom(codedInput);
codedInput.CheckReadEndOfStreamTag();
}

internal static void MergeDelimitedFrom(this IMessage message, Stream input, bool discardUnknownFields)
{
ProtoPreconditions.CheckNotNull(message, "message");
ProtoPreconditions.CheckNotNull(input, "input");
int size = (int) CodedInputStream.ReadRawVarint32(input);
Stream limitedStream = new LimitedInputStream(input, size);
MergeFrom(message, limitedStream, discardUnknownFields);
}
}
}
66 changes: 52 additions & 14 deletions csharp/src/Google.Protobuf/MessageParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,13 @@ namespace Google.Protobuf
public class MessageParser
{
private Func<IMessage> factory;
// TODO: When we use a C# 7.1 compiler, make this private protected.
internal bool DiscardUnknownFields { get; }

internal MessageParser(Func<IMessage> factory)
internal MessageParser(Func<IMessage> factory, bool discardUnknownFields)
{
this.factory = factory;
DiscardUnknownFields = discardUnknownFields;
}

/// <summary>
Expand All @@ -65,7 +68,7 @@ internal IMessage CreateTemplate()
public IMessage ParseFrom(byte[] data)
{
IMessage message = factory();
message.MergeFrom(data);
message.MergeFrom(data, DiscardUnknownFields);
return message;
}

Expand All @@ -79,7 +82,7 @@ public IMessage ParseFrom(byte[] data)
public IMessage ParseFrom(byte[] data, int offset, int length)
{
IMessage message = factory();
message.MergeFrom(data, offset, length);
message.MergeFrom(data, offset, length, DiscardUnknownFields);
return message;
}

Expand All @@ -91,7 +94,7 @@ public IMessage ParseFrom(byte[] data, int offset, int length)
public IMessage ParseFrom(ByteString data)
{
IMessage message = factory();
message.MergeFrom(data);
message.MergeFrom(data, DiscardUnknownFields);
return message;
}

Expand All @@ -103,7 +106,7 @@ public IMessage ParseFrom(ByteString data)
public IMessage ParseFrom(Stream input)
{
IMessage message = factory();
message.MergeFrom(input);
message.MergeFrom(input, DiscardUnknownFields);
return message;
}

Expand All @@ -119,7 +122,7 @@ public IMessage ParseFrom(Stream input)
public IMessage ParseDelimitedFrom(Stream input)
{
IMessage message = factory();
message.MergeDelimitedFrom(input);
message.MergeDelimitedFrom(input, DiscardUnknownFields);
return message;
}

Expand All @@ -131,7 +134,7 @@ public IMessage ParseDelimitedFrom(Stream input)
public IMessage ParseFrom(CodedInputStream input)
{
IMessage message = factory();
message.MergeFrom(input);
MergeFrom(message, input);
return message;
}

Expand All @@ -148,6 +151,29 @@ public IMessage ParseJson(string json)
JsonParser.Default.Merge(message, json);
return message;
}

// TODO: When we're using a C# 7.1 compiler, make this private protected.
internal void MergeFrom(IMessage message, CodedInputStream codedInput)
{
bool originalDiscard = codedInput.DiscardUnknownFields;
try
{
codedInput.DiscardUnknownFields = DiscardUnknownFields;
message.MergeFrom(codedInput);
}
finally
{
codedInput.DiscardUnknownFields = originalDiscard;
}
}

/// <summary>
/// Creates a new message parser which optionally discards unknown fields when parsing.
/// </summary>
/// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
/// <returns>A newly configured message parser.</returns>
public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) =>
new MessageParser(factory, discardUnknownFields);
}

/// <summary>
Expand Down Expand Up @@ -182,7 +208,11 @@ public sealed class MessageParser<T> : MessageParser where T : IMessage<T>
/// to require a parameterless constructor: delegates are significantly faster to execute.
/// </remarks>
/// <param name="factory">Function to invoke when a new, empty message is required.</param>
public MessageParser(Func<T> factory) : base(() => factory())
public MessageParser(Func<T> factory) : this(factory, false)
{
}

internal MessageParser(Func<T> factory, bool discardUnknownFields) : base(() => factory(), discardUnknownFields)
{
this.factory = factory;
}
Expand All @@ -204,7 +234,7 @@ internal new T CreateTemplate()
public new T ParseFrom(byte[] data)
{
T message = factory();
message.MergeFrom(data);
message.MergeFrom(data, DiscardUnknownFields);
return message;
}

Expand All @@ -218,7 +248,7 @@ public new T ParseFrom(byte[] data)
public new T ParseFrom(byte[] data, int offset, int length)
{
T message = factory();
message.MergeFrom(data, offset, length);
message.MergeFrom(data, offset, length, DiscardUnknownFields);
return message;
}

Expand All @@ -230,7 +260,7 @@ public new T ParseFrom(byte[] data, int offset, int length)
public new T ParseFrom(ByteString data)
{
T message = factory();
message.MergeFrom(data);
message.MergeFrom(data, DiscardUnknownFields);
return message;
}

Expand All @@ -242,7 +272,7 @@ public new T ParseFrom(ByteString data)
public new T ParseFrom(Stream input)
{
T message = factory();
message.MergeFrom(input);
message.MergeFrom(input, DiscardUnknownFields);
return message;
}

Expand All @@ -258,7 +288,7 @@ public new T ParseFrom(Stream input)
public new T ParseDelimitedFrom(Stream input)
{
T message = factory();
message.MergeDelimitedFrom(input);
message.MergeDelimitedFrom(input, DiscardUnknownFields);
return message;
}

Expand All @@ -270,7 +300,7 @@ public new T ParseDelimitedFrom(Stream input)
public new T ParseFrom(CodedInputStream input)
{
T message = factory();
message.MergeFrom(input);
MergeFrom(message, input);
return message;
}

Expand All @@ -287,5 +317,13 @@ public new T ParseJson(string json)
JsonParser.Default.Merge(message, json);
return message;
}

/// <summary>
/// Creates a new message parser which optionally discards unknown fields when parsing.
/// </summary>
/// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
/// <returns>A newly configured message parser.</returns>
public new MessageParser<T> WithDiscardUnknownFields(bool discardUnknownFields) =>
new MessageParser<T>(factory, discardUnknownFields);
}
}

0 comments on commit 47b7d2c

Please sign in to comment.