Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove BinaryFormatter from StateFileBase #6350

Merged
merged 8 commits into from May 24, 2021
Merged

Remove BinaryFormatter from StateFileBase #6350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8124add
Serialize state files without BinaryFormatter
Forgind Apr 15, 2021
013f65d
Add tests
Forgind Apr 15, 2021
708b7d9
Remove extra [Serializable]
Forgind Apr 15, 2021
fdc980a
Shift serialization to StateFileBase and extensions
Forgind Apr 16, 2021
fc25eb7
Add copyright header and remove fileExists
Forgind Apr 16, 2021
76b9923
Partially implement suggestions
Forgind Apr 20, 2021
858a99b
Remove Dependencies
Forgind Apr 20, 2021
69ad8b7
Comments. Also centralize version
Forgind Apr 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
54 changes: 48 additions & 6 deletions src/Shared/BinaryTranslator.cs
View file
Open in desktop
Expand Up @@ -660,12 +660,31 @@ public void TranslateDictionary<T>(ref Dictionary<string, T> dictionary, IEquali
}
}

/// <summary>
/// Reads in the boolean which says if this object is null or not.
/// </summary>
/// <typeparam name="T">The type of object to test.</typeparam>
/// <returns>True if the object should be read, false otherwise.</returns>
public bool TranslateNullable<T>(T value)
public void TranslateDictionary(ref Dictionary<string, DateTime> dictionary, StringComparer comparer)
{
if (!TranslateNullable(dictionary))
{
return;
}

int count = _reader.ReadInt32();
dictionary = new(count, comparer);
string key = string.Empty;
DateTime val = DateTime.MinValue;
for (int i = 0; i < count; i++)
{
Translate(ref key);
Translate(ref val);
danmoseley marked this conversation as resolved.
Show resolved Hide resolved
dictionary.Add(key, val);
}
}

/// <summary>
/// Reads in the boolean which says if this object is null or not.
/// </summary>
/// <typeparam name="T">The type of object to test.</typeparam>
/// <returns>True if the object should be read, false otherwise.</returns>
public bool TranslateNullable<T>(T value)
{
bool haveRef = _reader.ReadBoolean();
return haveRef;
Expand Down Expand Up @@ -1254,6 +1273,29 @@ public void TranslateDictionary<T>(ref Dictionary<string, T> dictionary, IEquali
}
}

/// <summary>
/// Translates a dictionary of { string, DateTime }.
/// </summary>
/// <param name="dictionary">The dictionary to be translated.</param>
/// <param name="comparer">Key comparer</param>
public void TranslateDictionary(ref Dictionary<string, DateTime> dictionary, StringComparer comparer)
{
if (!TranslateNullable(dictionary))
{
return;
}

int count = dictionary.Count;
_writer.Write(count);
foreach (KeyValuePair<string, DateTime> kvp in dictionary)
{
string key = kvp.Key;
DateTime val = kvp.Value;
Translate(ref key);
Translate(ref val);
}
}

/// <summary>
/// Writes out the boolean which says if this object is null or not.
/// </summary>
Expand Down
2 changes: 2 additions & 0 deletions src/Shared/ITranslator.cs
View file
Open in desktop
Expand Up @@ -301,6 +301,8 @@ void TranslateArray<T>(ref T[] array)

void TranslateDictionary(ref IDictionary<string, string> dictionary, NodePacketCollectionCreator<IDictionary<string, string>> collectionCreator);

void TranslateDictionary(ref Dictionary<string, DateTime> dictionary, StringComparer comparer);

void TranslateDictionary<K, V>(ref IDictionary<K, V> dictionary, ObjectTranslator<K> keyTranslator, ObjectTranslator<V> valueTranslator, NodePacketCollectionCreator<IDictionary<K, V>> dictionaryCreator);

/// <summary>
Expand Down
190 changes: 46 additions & 144 deletions src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs
View file
Open in desktop
@@ -1,6 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
rainersigwald marked this conversation as resolved.
Show resolved Hide resolved

using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Runtime.Versioning;
using Microsoft.Build.Shared;
using Microsoft.Build.Tasks;
Expand All @@ -12,10 +15,6 @@ namespace Microsoft.Build.UnitTests.ResolveAssemblyReference_Tests
{
public class ResolveAssemblyReferenceCacheSerialization : IDisposable
{
// Maintain this two in sync with the constant in SystemState
private static readonly byte[] TranslateContractSignature = { (byte)'M', (byte)'B', (byte)'R', (byte)'S', (byte)'C' }; // Microsoft Build RAR State Cache
private static readonly byte TranslateContractVersion = 0x01;

private readonly string _rarCacheFile;
private readonly TaskLoggingHelper _taskLoggingHelper;

Expand All @@ -42,178 +41,81 @@ public void RoundTripEmptyState()
{
SystemState systemState = new();

systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);

var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState));

deserialized.ShouldNotBeNull();
}

[Fact]
public void WrongFileSignature()
{
SystemState systemState = new();

for (int i = 0; i < TranslateContractSignature.Length; i++)
{
systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(i, SeekOrigin.Begin);
cacheStream.WriteByte(0);
cacheStream.Close();
}

var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);

deserialized.ShouldBeNull();
}
}

[Fact]
public void WrongFileVersion()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So is there now no equivalent to this test? What about the "cache was written with v.last and is read with v.current" scenario?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can reinstate it. To be honest, in retrospect, I'm not sure why I deleted it in the first place.

public void CorrectFileVersion()
{
SystemState systemState = new();

systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(TranslateContractSignature.Length, SeekOrigin.Begin);
cacheStream.WriteByte((byte) (TranslateContractVersion + 1));
cacheStream.Seek(0, SeekOrigin.Begin);
cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion);
cacheStream.Close();
}

var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);

deserialized.ShouldBeNull();
}

[Fact]
public void CorrectFileSignature()
{
SystemState systemState = new();

for (int i = 0; i < TranslateContractSignature.Length; i++)
{
systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(i, SeekOrigin.Begin);
cacheStream.WriteByte(TranslateContractSignature[i]);
cacheStream.Close();
}

var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState));

deserialized.ShouldNotBeNull();
}
deserialized.ShouldNotBeNull();
}

[Fact]
public void CorrectFileVersion()
public void WrongFileVersion()
{
SystemState systemState = new();

systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper);
using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite))
{
cacheStream.Seek(TranslateContractSignature.Length, SeekOrigin.Begin);
cacheStream.WriteByte(TranslateContractVersion);
cacheStream.Seek(0, SeekOrigin.Begin);
cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion - 1);
cacheStream.Close();
}

var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState));

deserialized.ShouldNotBeNull();
deserialized.ShouldBeNull();
}

[Fact]
public void VerifySampleStateDeserialization()
public void ValidateSerializationAndDeserialization()
{
// This test might also fail when binary format is modified.
// Any change in SystemState and child class ITranslatable implementation will most probably make this fail.
// To fix it, file referred by 'sampleName' needs to be recaptured and constant bellow modified to reflect
// the content of that cache.
// This sample was captured by compiling https://github.com/dotnet/roslyn/commit/f8107de2a94a01e96ac3d7c1f225acbb61e18830
const string sampleName = "Microsoft.VisualStudio.LanguageServices.Implementation.csprojAssemblyReference.cache";
const string expectedAssemblyPath = @"C:\Users\rokon\.nuget\packages\microsoft.visualstudio.codeanalysis.sdk.ui\15.8.27812-alpha\lib\net46\Microsoft.VisualStudio.CodeAnalysis.Sdk.UI.dll";
const long expectedAssemblyLastWriteTimeTicks = 636644382480000000;
const string expectedAssemblyName = "Microsoft.VisualStudio.CodeAnalysis.Sdk.UI, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a";
const string expectedFrameworkName = ".NETFramework,Version=v4.5";
var expectedDependencies = new[]
Dictionary<string, SystemState.FileState> cache = new() {
{ "path1", new SystemState.FileState(DateTime.Now) },
{ "path2", new SystemState.FileState(DateTime.Now) { Assembly = new AssemblyNameExtension("hi") } },
{ "dllName", new SystemState.FileState(DateTime.Now.AddSeconds(-10)) {
Assembly = null,
RuntimeVersion = "v4.0.30319",
FrameworkNameAttribute = new FrameworkName(".NETFramework", Version.Parse("4.7.2"), "Profile"),
scatterFiles = new string[] { "first", "second" } } } };
SystemState sysState = new();
sysState.instanceLocalFileStateCache = cache;
SystemState sysState2 = null;
using (TestEnvironment env = TestEnvironment.Create())
{
"mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.CodeAnalysis, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.DeveloperTools, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"Microsoft.VisualStudio.Shell.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.CodeAnalysis.Sdk, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.Build.Framework, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Text.Logic, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Text.UI, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Text.Data, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Text.UI.Wpf, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.ComponentModelHost, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.VSHelp, Version=7.0.3300.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Shell.Interop.11.0, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.VCProjectEngine, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Shell.15.0, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.OLE.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"System.Xml, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"Microsoft.VisualStudio.TextManager.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"System.Core, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089",
"Microsoft.VisualStudio.VirtualTreeGrid, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Shell.Interop.8.0, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
"Microsoft.VisualStudio.Editor, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
};


CopyResourceSampleFileIntoRarCacheFile($@"AssemblyDependency\CacheFileSamples\{sampleName}");

var deserializedByTranslator = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper);
deserializedByTranslator.ShouldNotBeNull();

deserializedByTranslator.SetGetLastWriteTime(path =>
{
if (path != expectedAssemblyPath)
throw new InvalidOperationException("Unexpected file name for this test case");

return new DateTime(expectedAssemblyLastWriteTimeTicks, DateTimeKind.Utc);
});

GetAssemblyName getAssemblyName = deserializedByTranslator.CacheDelegate((GetAssemblyName)null);
GetAssemblyMetadata getAssemblyMetadata = deserializedByTranslator.CacheDelegate((GetAssemblyMetadata)null);

var assemblyName = getAssemblyName(expectedAssemblyPath);
getAssemblyMetadata(expectedAssemblyPath, null,
out AssemblyNameExtension[] dependencies,
out string[] scatterFiles,
out FrameworkName frameworkNameAttribute);


assemblyName.ShouldNotBeNull();
assemblyName.ShouldBe(new AssemblyNameExtension(expectedAssemblyName, false));
scatterFiles.ShouldBeEmpty();
frameworkNameAttribute.ShouldBe(new FrameworkName(expectedFrameworkName));
dependencies.ShouldNotBeNull();
expectedDependencies.ShouldBe(expectedDependencies, ignoreOrder: true);
}

private void CopyResourceSampleFileIntoRarCacheFile(string name)
{
Assembly asm = this.GetType().Assembly;
var resource = string.Format($"{asm.GetName().Name}.{name.Replace("\\", ".")}");
using Stream resourceStream = asm.GetManifestResourceStream(resource);
if (resourceStream == null)
throw new InvalidOperationException($"Resource '{resource}' has not been found.");

using FileStream rarCacheFile = new FileStream(_rarCacheFile, FileMode.CreateNew);
TransientTestFile file = env.CreateFile();
sysState.SerializeCache(file.Path, null);
sysState2 = SystemState.DeserializeCache(file.Path, null, typeof(SystemState)) as SystemState;
}

resourceStream.CopyTo(rarCacheFile);
Dictionary<string, SystemState.FileState> cache2 = sysState2.instanceLocalFileStateCache;
cache2.Count.ShouldBe(cache.Count);
cache2["path2"].Assembly.Name.ShouldBe(cache["path2"].Assembly.Name);
SystemState.FileState dll = cache["dllName"];
SystemState.FileState dll2 = cache2["dllName"];
dll2.Assembly.ShouldBe(dll.Assembly);
dll2.FrameworkNameAttribute.FullName.ShouldBe(dll.FrameworkNameAttribute.FullName);
dll2.LastModified.ShouldBe(dll.LastModified);
dll2.RuntimeVersion.ShouldBe(dll.RuntimeVersion);
dll2.scatterFiles.Length.ShouldBe(dll.scatterFiles.Length);
dll2.scatterFiles[1].ShouldBe(dll.scatterFiles[1]);
}
}
}
20 changes: 20 additions & 0 deletions src/Tasks.UnitTests/AssemblyRegistrationCache_Tests.cs
View file
Open in desktop
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using Microsoft.Build.Tasks;
using Shouldly;
using Xunit;

namespace Microsoft.Build.UnitTests
Expand All @@ -26,5 +27,24 @@ public void ExerciseCache()
Assert.Equal("foo", assembly);
Assert.Equal("bar", tlb);
}

[Fact]
public void ExerciseCacheSerialization()
{
AssemblyRegistrationCache arc = new();
arc.AddEntry("foo", "bar");
AssemblyRegistrationCache arc2 = null;
using (TestEnvironment env = TestEnvironment.Create())
{
TransientTestFile file = env.CreateFile();
arc.SerializeCache(file.Path, null);
arc2 = StateFileBase.DeserializeCache(file.Path, null, typeof(AssemblyRegistrationCache)) as AssemblyRegistrationCache;
}

arc2._assemblies.Count.ShouldBe(arc._assemblies.Count);
arc2._assemblies[0].ShouldBe(arc._assemblies[0]);
arc2._typeLibraries.Count.ShouldBe(arc._typeLibraries.Count);
arc2._typeLibraries[0].ShouldBe(arc._typeLibraries[0]);
}
}
}
26 changes: 26 additions & 0 deletions src/Tasks.UnitTests/ResolveComReference_Tests.cs
View file
Open in desktop
Expand Up @@ -14,6 +14,9 @@
using Microsoft.Build.Tasks;
using Xunit;
using Microsoft.Build.Shared;
using System.IO;
using Microsoft.Build.BackEnd;
using Shouldly;

namespace Microsoft.Build.UnitTests
{
Expand Down Expand Up @@ -57,6 +60,29 @@ public void GetResolvedASsemblyReferenceSpecNotNull()
Assert.NotNull(task.GetResolvedAssemblyReferenceItemSpecs());
}

[Fact]
public void TestSerializationAndDeserialization()
{
ResolveComReferenceCache cache = new("path1", "path2");
cache.componentTimestamps = new()
{
{ "first", DateTime.Now },
{ "second", DateTime.FromBinary(10000) },
};
ResolveComReferenceCache cache2 = null;
using (TestEnvironment env = TestEnvironment.Create())
{
TransientTestFile file = env.CreateFile();
cache.SerializeCache(file.Path, null);
cache2 = StateFileBase.DeserializeCache(file.Path, null, typeof(ResolveComReferenceCache)) as ResolveComReferenceCache;
}

cache2.tlbImpLocation.ShouldBe(cache.tlbImpLocation);
cache2.axImpLocation.ShouldBe(cache.axImpLocation);
cache2.componentTimestamps.Count.ShouldBe(cache.componentTimestamps.Count);
cache2.componentTimestamps["second"].ShouldBe(cache.componentTimestamps["second"]);
}

/*
* Method: CheckComReferenceAttributeVerificationForNameItems
*
Expand Down