Skip to content

Commit

Permalink
On disk cache serialization (#6094)
Browse files Browse the repository at this point in the history 
### Context
BinaryFormater serialization of on-disk RAR cache is slow and unsecure. 
Related issue: #6057

### Changes Made
Serialization changed to use custom binary format by using existing `ITranslatable` 

### Testing
Unit testing
Manual testing
- Roslyn repo rebuild + incremental build
  • Loading branch information
@rokonec
rokonec committed Feb 25, 2021
1 parent 891b99c commit 6819f7a
Show file tree
Hide file tree
Showing 13 changed files with 887 additions and 72 deletions.
179 changes: 179 additions & 0 deletions src/Build.UnitTests/BackEnd/BinaryTranslator_Tests.cs
View file
Expand Up @@ -3,8 +3,12 @@

using System;
using System.Collections.Generic;
using System.Configuration.Assemblies;
using System.Globalization;
using Microsoft.Build.BackEnd;
using System.IO;
using System.Reflection;
using Shouldly;
using Xunit;

namespace Microsoft.Build.UnitTests.BackEnd
Expand Down Expand Up @@ -431,6 +435,163 @@ public void TestSerializeDictionaryStringTNoComparerNull()
Assert.Equal(value, deserializedValue);
}

[Theory]
[InlineData("en")]
[InlineData("en-US")]
[InlineData("en-CA")]
[InlineData("zh-HK")]
[InlineData("sr-Cyrl-CS")]
public void CultureInfo(string name)
{
CultureInfo value = new CultureInfo(name);
TranslationHelpers.GetWriteTranslator().Translate(ref value);

CultureInfo deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

deserializedValue.ShouldBe(value);
}

[Fact]
public void CultureInfoAsNull()
{
CultureInfo value = null;
TranslationHelpers.GetWriteTranslator().Translate(ref value);

CultureInfo deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

deserializedValue.ShouldBeNull();
}

[Theory]
[InlineData("1.2")]
[InlineData("1.2.3")]
[InlineData("1.2.3.4")]
public void Version(string version)
{
Version value = new Version(version);
TranslationHelpers.GetWriteTranslator().Translate(ref value);

Version deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

deserializedValue.ShouldBe(value);
}

[Fact]
public void VersionAsNull()
{
Version value = null;
TranslationHelpers.GetWriteTranslator().Translate(ref value);

Version deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

deserializedValue.ShouldBeNull();
}

[Fact]
public void HashSetOfT()
{
HashSet<BaseClass> values = new()
{
new BaseClass(1),
new BaseClass(2),
null
};
TranslationHelpers.GetWriteTranslator().TranslateHashSet(ref values, BaseClass.FactoryForDeserialization, capacity => new());

HashSet<BaseClass> deserializedValues = null;
TranslationHelpers.GetReadTranslator().TranslateHashSet(ref deserializedValues, BaseClass.FactoryForDeserialization, capacity => new());

deserializedValues.ShouldBe(values, ignoreOrder: true);
}

[Fact]
public void HashSetOfTAsNull()
{
HashSet<BaseClass> value = null;
TranslationHelpers.GetWriteTranslator().TranslateHashSet(ref value, BaseClass.FactoryForDeserialization, capacity => new());

HashSet<BaseClass> deserializedValue = null;
TranslationHelpers.GetReadTranslator().TranslateHashSet(ref deserializedValue, BaseClass.FactoryForDeserialization, capacity => new());

deserializedValue.ShouldBeNull();
}

[Fact]
public void AssemblyNameAsNull()
{
AssemblyName value = null;
TranslationHelpers.GetWriteTranslator().Translate(ref value);

AssemblyName deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

deserializedValue.ShouldBeNull();
}

[Fact]
public void AssemblyNameWithAllFields()
{
AssemblyName value = new()
{
Name = "a",
Version = new Version(1, 2, 3),
Flags = AssemblyNameFlags.PublicKey,
ProcessorArchitecture = ProcessorArchitecture.X86,
CultureInfo = new CultureInfo("zh-HK"),
HashAlgorithm = System.Configuration.Assemblies.AssemblyHashAlgorithm.SHA256,
VersionCompatibility = AssemblyVersionCompatibility.SameMachine,
CodeBase = "C:\\src",
KeyPair = new StrongNameKeyPair(new byte[] { 4, 3, 2, 1 }),
ContentType = AssemblyContentType.WindowsRuntime,
CultureName = "zh-HK",
};
value.SetPublicKey(new byte[]{ 3, 2, 1});
value.SetPublicKeyToken(new byte[] { 8, 7, 6, 5, 4, 3, 2, 1 });

TranslationHelpers.GetWriteTranslator().Translate(ref value);

AssemblyName deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

HelperAssertAssemblyNameEqual(value, deserializedValue);
}

[Fact]
public void AssemblyNameWithMinimalFields()
{
AssemblyName value = new();

TranslationHelpers.GetWriteTranslator().Translate(ref value);

AssemblyName deserializedValue = null;
TranslationHelpers.GetReadTranslator().Translate(ref deserializedValue);

HelperAssertAssemblyNameEqual(value, deserializedValue);
}

/// <summary>
/// Assert two AssemblyName objects values are same.
/// Ignoring KeyPair, ContentType, CultureName as those are not serialized
/// </summary>
private static void HelperAssertAssemblyNameEqual(AssemblyName expected, AssemblyName actual)
{
actual.Name.ShouldBe(expected.Name);
actual.Version.ShouldBe(expected.Version);
actual.Flags.ShouldBe(expected.Flags);
actual.ProcessorArchitecture.ShouldBe(expected.ProcessorArchitecture);
actual.CultureInfo.ShouldBe(expected.CultureInfo);
actual.HashAlgorithm.ShouldBe(expected.HashAlgorithm);
actual.VersionCompatibility.ShouldBe(expected.VersionCompatibility);
actual.CodeBase.ShouldBe(expected.CodeBase);

actual.GetPublicKey().ShouldBe(expected.GetPublicKey());
actual.GetPublicKeyToken().ShouldBe(expected.GetPublicKeyToken());
}

/// <summary>
/// Helper for bool serialization.
/// </summary>
Expand Down Expand Up @@ -610,6 +771,24 @@ protected BaseClass()
{
}

protected bool Equals(BaseClass other)
{
return _baseValue == other._baseValue;
}

public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
if (ReferenceEquals(this, obj)) return true;
if (obj.GetType() != this.GetType()) return false;
return Equals((BaseClass) obj);
}

public override int GetHashCode()
{
return _baseValue;
}

/// <summary>
/// Gets a comparer.
/// </summary>
Expand Down
39 changes: 37 additions & 2 deletions src/Shared/AssemblyNameExtension.cs
View file
Expand Up @@ -9,6 +9,7 @@
using System.Configuration.Assemblies;
using System.Runtime.Serialization;
using System.IO;
using Microsoft.Build.BackEnd;
#if FEATURE_ASSEMBLYLOADCONTEXT
using System.Reflection.PortableExecutable;
using System.Reflection.Metadata;
Expand Down Expand Up @@ -54,7 +55,7 @@ internal enum PartialComparisonFlags : int
/// between the two is done lazily on demand.
/// </summary>
[Serializable]
internal sealed class AssemblyNameExtension : ISerializable, IEquatable<AssemblyNameExtension>
internal sealed class AssemblyNameExtension : ISerializable, IEquatable<AssemblyNameExtension>, ITranslatable
{
private AssemblyName asAssemblyName = null;
private string asString = null;
Expand Down Expand Up @@ -173,6 +174,14 @@ private AssemblyNameExtension(SerializationInfo info, StreamingContext context)
remappedFrom = (HashSet<AssemblyNameExtension>) info.GetValue("remapped", typeof(HashSet<AssemblyNameExtension>));
}

/// <summary>
/// Ctor for deserializing from state file (custom binary serialization) using translator.
/// </summary>
internal AssemblyNameExtension(ITranslator translator) : this()
{
Translate(translator);
}

/// <summary>
/// To be used as a delegate. Gets the AssemblyName of the given file.
/// </summary>
Expand Down Expand Up @@ -251,10 +260,18 @@ private void InitializeRemappedFrom()
{
if (remappedFrom == null)
{
remappedFrom = new HashSet<AssemblyNameExtension>(AssemblyNameComparer.GenericComparerConsiderRetargetable);
remappedFrom = CreateRemappedFrom();
}
}

/// <summary>
/// Create remappedFrom HashSet. Used by deserialization as well.
/// </summary>
private static HashSet<AssemblyNameExtension> CreateRemappedFrom()
{
return new HashSet<AssemblyNameExtension>(AssemblyNameComparer.GenericComparerConsiderRetargetable);
}

/// <summary>
/// Assume there is a string version, create the AssemblyName version.
/// </summary>
Expand Down Expand Up @@ -993,5 +1010,23 @@ public void GetObjectData(SerializationInfo info, StreamingContext context)
info.AddValue("immutable", immutable);
info.AddValue("remapped", remappedFrom);
}

/// <summary>
/// Reads/writes this class
/// </summary>
/// <param name="translator"></param>
public void Translate(ITranslator translator)
{
translator.Translate(ref asAssemblyName);
translator.Translate(ref asString);
translator.Translate(ref isSimpleName);
translator.Translate(ref hasProcessorArchitectureInFusionName);
translator.Translate(ref immutable);

// TODO: consider some kind of protection against infinite loop during serialization, hint: pre serialize check for cycle in graph
translator.TranslateHashSet(ref remappedFrom,
(ITranslator t) => new AssemblyNameExtension(t),
(int capacity) => CreateRemappedFrom());
}
}
}

0 comments on commit 6819f7a

Please sign in to comment.