From 8124add656e4bb38017302f1efbc1780c5c4f724 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Thu, 15 Apr 2021 10:15:53 -0700 Subject: [PATCH 1/8] Serialize state files without BinaryFormatter --- src/Tasks/AssemblyRegistrationCache.cs | 6 +- src/Tasks/Dependencies.cs | 12 ++-- src/Tasks/DependencyFile.cs | 13 ++-- src/Tasks/ResGenDependencies.cs | 26 +++++--- src/Tasks/ResolveComReferenceCache.cs | 17 +++-- src/Tasks/StateFileBase.cs | 46 +++++-------- src/Tasks/SystemState.cs | 2 - src/Tasks/TaskTranslatorHelpers.cs | 90 ++++++++++++++++++++++++++ 8 files changed, 143 insertions(+), 69 deletions(-) diff --git a/src/Tasks/AssemblyRegistrationCache.cs b/src/Tasks/AssemblyRegistrationCache.cs index 63ae6fdd833..eb1f0eda8c3 100644 --- a/src/Tasks/AssemblyRegistrationCache.cs +++ b/src/Tasks/AssemblyRegistrationCache.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.Collections.Generic; using Microsoft.Build.Shared; @@ -10,18 +9,17 @@ namespace Microsoft.Build.Tasks /// /// This class is a caching mechanism for the Register/UnregisterAssembly task to keep track of registered assemblies to clean up /// - [Serializable()] internal sealed class AssemblyRegistrationCache : StateFileBase { /// /// The list of registered assembly files. /// - private readonly List _assemblies = new List(); + internal List _assemblies = new List(); /// /// The list of registered type library files. /// - private readonly List _typeLibraries = new List(); + internal List _typeLibraries = new List(); /// /// The number of entries in the state file diff --git a/src/Tasks/Dependencies.cs b/src/Tasks/Dependencies.cs index 80761d7464f..aac1fc0b13b 100644 --- a/src/Tasks/Dependencies.cs +++ b/src/Tasks/Dependencies.cs @@ -1,32 +1,30 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Collections; +using System.Collections.Generic; namespace Microsoft.Build.Tasks { /// /// Represents a cache of inputs to a compilation-style task. /// - /// On-disk serialization format, don't change field names or types or use readonly. - [Serializable] internal class Dependencies { /// /// Hashtable of other dependency files. /// Key is filename and value is DependencyFile. /// - private Hashtable dependencies = new Hashtable(); + internal Dictionary dependencies = new(); /// - /// Look up a dependency file. Return null if its not there. + /// Look up a dependency file. Return null if it isn't there. /// /// /// internal DependencyFile GetDependencyFile(string filename) { - return (DependencyFile)dependencies[filename]; + dependencies.TryGetValue(filename, out DependencyFile file); + return file; } /// diff --git a/src/Tasks/DependencyFile.cs b/src/Tasks/DependencyFile.cs index f8306894c3b..1851506118d 100644 --- a/src/Tasks/DependencyFile.cs +++ b/src/Tasks/DependencyFile.cs @@ -12,20 +12,17 @@ namespace Microsoft.Build.Tasks /// /// Represents a single input to a compilation-style task. /// Keeps track of timestamp for later comparison. - /// - /// On-disk serialization format, don't change field names or types or use readonly. /// - [Serializable] internal class DependencyFile { // Filename - private string filename; + internal string filename; // Date and time the file was last modified - private DateTime lastModified; + internal DateTime lastModified; // Whether the file exists or not. - private bool exists = false; + internal bool exists = false; /// /// The name of the file. @@ -70,6 +67,10 @@ internal DependencyFile(string filename) } } + internal DependencyFile() + { + } + /// /// Checks whether the file has changed since the last time a timestamp was recorded. /// diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index 3632a916d83..e86e81f4e50 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -22,18 +22,17 @@ namespace Microsoft.Build.Tasks /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - [Serializable] internal sealed class ResGenDependencies : StateFileBase { /// /// The list of resx files. /// - private Dependencies resXFiles = new Dependencies(); + internal Dependencies resXFiles = new Dependencies(); /// /// A list of portable libraries and the ResW files they can produce. /// - private Dependencies portableLibraries = new Dependencies(); + internal Dependencies portableLibraries = new Dependencies(); /// /// A newly-created ResGenDependencies is not dirty. @@ -47,7 +46,7 @@ internal sealed class ResGenDependencies : StateFileBase /// If this is NULL then we use the directory in which the .resx is in (that should always /// be the default!) /// - private string baseLinkedFileDirectory; + internal string baseLinkedFileDirectory; internal string BaseLinkedFileDirectory { @@ -93,8 +92,7 @@ internal bool UseSourcePath internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) { // First, try to retrieve the resx information from our hashtable. - var retVal = (ResXFile)resXFiles.GetDependencyFile(resxFile); - if (retVal == null) + if (resXFiles.GetDependencyFile(resxFile) is not ResXFile retVal || retVal == null) { // Ok, the file wasn't there. Add it to our cache and return it to the caller. retVal = AddResxFile(resxFile, useMSBuildResXReader); @@ -192,7 +190,7 @@ internal static ResGenDependencies DeserializeCache(string stateFile, bool useSo internal sealed class ResXFile : DependencyFile { // Files contained within this resx file. - private string[] linkedFiles; + internal string[] linkedFiles; internal string[] LinkedFiles => linkedFiles; @@ -209,6 +207,10 @@ internal ResXFile(string filename, string baseLinkedFileDirectory, bool useMSBui } } + internal ResXFile() + { + } + /// /// Given a .RESX file, returns all the linked files that are referenced within that .RESX. /// @@ -284,9 +286,13 @@ private static string[] GetLinkedFiles(string filename, string baseLinkedFileDir [Serializable] internal sealed class PortableLibraryFile : DependencyFile { - private string[] outputFiles; - private string neutralResourceLanguage; - private string assemblySimpleName; + internal string[] outputFiles; + internal string neutralResourceLanguage; + internal string assemblySimpleName; + + internal PortableLibraryFile() + { + } internal PortableLibraryFile(string filename) : base(filename) diff --git a/src/Tasks/ResolveComReferenceCache.cs b/src/Tasks/ResolveComReferenceCache.cs index d0f80dca18f..87fa0210464 100644 --- a/src/Tasks/ResolveComReferenceCache.cs +++ b/src/Tasks/ResolveComReferenceCache.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections; +using System.Collections.Generic; using Microsoft.Build.Shared; namespace Microsoft.Build.Tasks @@ -16,7 +16,6 @@ namespace Microsoft.Build.Tasks /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - [Serializable] internal sealed class ResolveComReferenceCache : StateFileBase { /// @@ -24,9 +23,9 @@ internal sealed class ResolveComReferenceCache : StateFileBase /// Key: Component path on disk /// Value: DateTime struct /// - private Hashtable componentTimestamps; - private string tlbImpLocation; - private string axImpLocation; + internal Dictionary componentTimestamps; + internal string tlbImpLocation; + internal string axImpLocation; /// /// indicates whether the cache contents have changed since it's been created @@ -46,7 +45,7 @@ internal ResolveComReferenceCache(string tlbImpPath, string axImpPath) tlbImpLocation = tlbImpPath; axImpLocation = axImpPath; - componentTimestamps = new Hashtable(); + componentTimestamps = new(); } /// @@ -69,9 +68,9 @@ internal bool ToolPathsMatchCachePaths(string tlbImpPath, string axImpPath) { get { - if (componentTimestamps.ContainsKey(componentPath)) + if (componentTimestamps.TryGetValue(componentPath, out DateTime time)) { - return (DateTime)componentTimestamps[componentPath]; + return time; } // If the entry is not present in the cache, return the current time. Since no component should be timestamped @@ -81,7 +80,7 @@ internal bool ToolPathsMatchCachePaths(string tlbImpPath, string axImpPath) set { // only set the value and dirty the cache if the timestamp doesn't exist yet or is different than the current one - if (DateTime.Compare(this[componentPath], value) != 0) + if (!DateTime.Equals(this[componentPath], value)) { componentTimestamps[componentPath] = value; _dirty = true; diff --git a/src/Tasks/StateFileBase.cs b/src/Tasks/StateFileBase.cs index b5b34a31b0b..01ec834fa4d 100644 --- a/src/Tasks/StateFileBase.cs +++ b/src/Tasks/StateFileBase.cs @@ -3,17 +3,16 @@ using System; using System.IO; -using System.Runtime.Serialization.Formatters.Binary; using Microsoft.Build.Utilities; using Microsoft.Build.Shared; using Microsoft.Build.Shared.FileSystem; +using Microsoft.Build.BackEnd; namespace Microsoft.Build.Tasks { /// /// Base class for task state files. /// - [Serializable()] internal class StateFileBase { // Current version for serialization. This should be changed when breaking changes @@ -43,18 +42,16 @@ internal virtual void SerializeCache(string stateFile, TaskLoggingHelper log) using (var s = new FileStream(stateFile, FileMode.CreateNew)) { - var formatter = new BinaryFormatter(); - formatter.Serialize(s, this); + var translator = BinaryTranslator.GetWriteTranslator(s); + StateFileBase thisCopy = this; + translator.Translate(ref thisCopy, thisCopy.GetType()); } } } - catch (Exception e) + // If there was a problem writing the file (like it's read-only or locked on disk, for + // example), then eat the exception and log a warning. Otherwise, rethrow. + catch (Exception e) when (!ExceptionHandling.NotExpectedSerializationException(e)) { - // If there was a problem writing the file (like it's read-only or locked on disk, for - // example), then eat the exception and log a warning. Otherwise, rethrow. - if (ExceptionHandling.NotExpectedSerializationException(e)) - throw; - // Not being able to serialize the cache is not an error, but we let the user know anyway. // Don't want to hold up processing just because we couldn't read the file. log.LogWarningWithCodeFromResources("General.CouldNotWriteStateFile", stateFile, e.Message); @@ -75,43 +72,30 @@ internal static StateFileBase DeserializeCache(string stateFile, TaskLoggingHelp { using (FileStream s = new FileStream(stateFile, FileMode.Open)) { - var formatter = new BinaryFormatter(); - object deserializedObject = formatter.Deserialize(s); - retVal = deserializedObject as StateFileBase; + var translator = BinaryTranslator.GetReadTranslator(s, buffer: null); + translator.Translate(ref retVal, requiredReturnType); - // If the deserialized object is null then there would be no cast error but retVal would still be null - // only log the message if there would have been a cast error - if (retVal == null && deserializedObject != null) + // If retVal is still null or the version is wrong, log a message not a warning. This could be a valid cache with the wrong version preventing correct deserialization. + // For the latter case, internals may be unexpectedly null. + if (retVal == null || retVal._serializedVersion != CurrentSerializationVersion) { // When upgrading to Visual Studio 2008 and running the build for the first time the resource cache files are replaced which causes a cast error due // to a new version number on the tasks class. "Unable to cast object of type 'Microsoft.Build.Tasks.SystemState' to type 'Microsoft.Build.Tasks.StateFileBase". // If there is an invalid cast, a message rather than a warning should be emitted. log.LogMessageFromResources("General.CouldNotReadStateFileMessage", stateFile, log.FormatResourceString("General.IncompatibleStateFileType")); + return null; } - - if ((retVal != null) && (!requiredReturnType.IsInstanceOfType(retVal))) + else if (!requiredReturnType.IsInstanceOfType(retVal)) { log.LogMessageFromResources("General.CouldNotReadStateFileMessage", stateFile, log.FormatResourceString("General.IncompatibleStateFileType")); retVal = null; } - - // If we get back a valid object and internals were changed, things are likely to be null. Check the version before we use it. - if (retVal != null && retVal._serializedVersion != CurrentSerializationVersion) - { - log.LogMessageFromResources("General.CouldNotReadStateFileMessage", stateFile, log.FormatResourceString("General.IncompatibleStateFileType")); - retVal = null; - } } } } - catch (Exception e) + catch (Exception e) when (!ExceptionHandling.IsCriticalException(e)) { - if (ExceptionHandling.IsCriticalException(e)) - { - throw; - } - // The deserialization process seems like it can throw just about // any exception imaginable. Catch them all here. // Not being able to deserialize the cache is not an error, but we let the user know anyway. diff --git a/src/Tasks/SystemState.cs b/src/Tasks/SystemState.cs index 3f5ea428b0c..d121def739c 100644 --- a/src/Tasks/SystemState.cs +++ b/src/Tasks/SystemState.cs @@ -21,7 +21,6 @@ namespace Microsoft.Build.Tasks /// /// Class is used to cache system state. /// - [Serializable] internal sealed class SystemState : StateFileBase, ITranslatable { private static readonly byte[] TranslateContractSignature = { (byte) 'M', (byte) 'B', (byte) 'R', (byte) 'S', (byte) 'C'}; // Microsoft Build RAR State Cache @@ -112,7 +111,6 @@ internal sealed class SystemState : StateFileBase, ITranslatable /// /// Class that holds the current file state. /// - [Serializable] internal sealed class FileState : ITranslatable { /// diff --git a/src/Tasks/TaskTranslatorHelpers.cs b/src/Tasks/TaskTranslatorHelpers.cs index 7db48cf61ae..c5bb030a4e4 100644 --- a/src/Tasks/TaskTranslatorHelpers.cs +++ b/src/Tasks/TaskTranslatorHelpers.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.Runtime.Versioning; using Microsoft.Build.BackEnd; @@ -34,5 +35,94 @@ public static void Translate(this ITranslator translator, ref FrameworkName fram frameworkName = new FrameworkName(identifier, version, profile); } } + + public static void TranslateDictionary(this ITranslator translator, ref Dictionary dict, StringComparer comparer) + { + int count = 0; + if (translator.Mode == TranslationDirection.ReadFromStream) + { + dict = new Dictionary(comparer); + translator.Translate(ref count); + string key = string.Empty; + DateTime val = DateTime.Now; + for (int i = 0; i < count; i++) + { + translator.Translate(ref key); + translator.Translate(ref val); + dict.Add(key, val); + } + } + else + { + count = dict.Count; + translator.Translate(ref count); + foreach (KeyValuePair kvp in dict) + { + string key = kvp.Key; + DateTime val = kvp.Value; + translator.Translate(ref key); + translator.Translate(ref val); + } + } + } + + public static void Translate(this ITranslator translator, ref Dependencies dependencies, Type t) + { + translator.TranslateDictionary(ref dependencies.dependencies, (ITranslator translator, ref DependencyFile dependency) => { + if (t == typeof(ResGenDependencies.ResXFile)) + { + ResGenDependencies.ResXFile resx = dependency as ResGenDependencies.ResXFile; + resx ??= new(); + translator.Translate(ref resx.linkedFiles); + dependency = resx; + } + else if (t == typeof(ResGenDependencies.PortableLibraryFile)) + { + ResGenDependencies.PortableLibraryFile lib = dependency as ResGenDependencies.PortableLibraryFile; + lib ??= new(); + translator.Translate(ref lib.assemblySimpleName); + translator.Translate(ref lib.outputFiles); + translator.Translate(ref lib.neutralResourceLanguage); + dependency = lib; + } + + dependency ??= new(); + translator.Translate(ref dependency.filename); + translator.Translate(ref dependency.lastModified); + translator.Translate(ref dependency.exists); + }); + } + + public static void Translate(this ITranslator translator, ref StateFileBase stateFile, Type t) + { + if (t == typeof(ResGenDependencies)) + { + ResGenDependencies rgd = stateFile as ResGenDependencies; + rgd ??= new(); + translator.Translate(ref rgd.resXFiles, typeof(ResGenDependencies.ResXFile)); + translator.Translate(ref rgd.portableLibraries, typeof(ResGenDependencies.PortableLibraryFile)); + translator.Translate(ref rgd.baseLinkedFileDirectory); + stateFile = rgd; + } +#if NETFRAMEWORK + else if (t == typeof(ResolveComReferenceCache)) + { + ResolveComReferenceCache rcrc = stateFile as ResolveComReferenceCache; + rcrc ??= new(string.Empty, string.Empty); + translator.Translate(ref rcrc.axImpLocation); + translator.Translate(ref rcrc.tlbImpLocation); + translator.TranslateDictionary(ref rcrc.componentTimestamps, StringComparer.Ordinal); + stateFile = rcrc; + } + else if (t == typeof(AssemblyRegistrationCache)) + { + AssemblyRegistrationCache arc = stateFile as AssemblyRegistrationCache; + arc ??= new(); + translator.Translate(ref arc._assemblies); + translator.Translate(ref arc._typeLibraries); + stateFile = arc; + } +#endif + } } } From 013f65dc2f3d865dcde9f6dada72e89367a6da93 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Thu, 15 Apr 2021 10:15:57 -0700 Subject: [PATCH 2/8] Add tests --- .../AssemblyRegistrationCache_Tests.cs | 20 +++++++ .../ResolveComReference_Tests.cs | 26 +++++++++ .../ResGenDependencies_Tests.cs | 55 +++++++++++++++---- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/src/Tasks.UnitTests/AssemblyRegistrationCache_Tests.cs b/src/Tasks.UnitTests/AssemblyRegistrationCache_Tests.cs index d496e33ecf1..fc94fbb510c 100644 --- a/src/Tasks.UnitTests/AssemblyRegistrationCache_Tests.cs +++ b/src/Tasks.UnitTests/AssemblyRegistrationCache_Tests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Build.Tasks; +using Shouldly; using Xunit; namespace Microsoft.Build.UnitTests @@ -26,5 +27,24 @@ public void ExerciseCache() Assert.Equal("foo", assembly); Assert.Equal("bar", tlb); } + + [Fact] + public void ExerciseCacheSerialization() + { + AssemblyRegistrationCache arc = new(); + arc.AddEntry("foo", "bar"); + AssemblyRegistrationCache arc2 = null; + using (TestEnvironment env = TestEnvironment.Create()) + { + TransientTestFile file = env.CreateFile(); + arc.SerializeCache(file.Path, null); + arc2 = StateFileBase.DeserializeCache(file.Path, null, typeof(AssemblyRegistrationCache)) as AssemblyRegistrationCache; + } + + arc2._assemblies.Count.ShouldBe(arc._assemblies.Count); + arc2._assemblies[0].ShouldBe(arc._assemblies[0]); + arc2._typeLibraries.Count.ShouldBe(arc._typeLibraries.Count); + arc2._typeLibraries[0].ShouldBe(arc._typeLibraries[0]); + } } } diff --git a/src/Tasks.UnitTests/ResolveComReference_Tests.cs b/src/Tasks.UnitTests/ResolveComReference_Tests.cs index f8c1c4855f7..4e695b86a73 100644 --- a/src/Tasks.UnitTests/ResolveComReference_Tests.cs +++ b/src/Tasks.UnitTests/ResolveComReference_Tests.cs @@ -14,6 +14,9 @@ using Microsoft.Build.Tasks; using Xunit; using Microsoft.Build.Shared; +using System.IO; +using Microsoft.Build.BackEnd; +using Shouldly; namespace Microsoft.Build.UnitTests { @@ -57,6 +60,29 @@ public void GetResolvedASsemblyReferenceSpecNotNull() Assert.NotNull(task.GetResolvedAssemblyReferenceItemSpecs()); } + [Fact] + public void TestSerializationAndDeserialization() + { + ResolveComReferenceCache cache = new("path1", "path2"); + cache.componentTimestamps = new() + { + { "first", DateTime.Now }, + { "second", DateTime.FromBinary(10000) }, + }; + ResolveComReferenceCache cache2 = null; + using (TestEnvironment env = TestEnvironment.Create()) + { + TransientTestFile file = env.CreateFile(); + cache.SerializeCache(file.Path, null); + cache2 = StateFileBase.DeserializeCache(file.Path, null, typeof(ResolveComReferenceCache)) as ResolveComReferenceCache; + } + + cache2.tlbImpLocation.ShouldBe(cache.tlbImpLocation); + cache2.axImpLocation.ShouldBe(cache.axImpLocation); + cache2.componentTimestamps.Count.ShouldBe(cache.componentTimestamps.Count); + cache2.componentTimestamps["second"].ShouldBe(cache.componentTimestamps["second"]); + } + /* * Method: CheckComReferenceAttributeVerificationForNameItems * diff --git a/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs b/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs index a48675078f4..bb4ca7de48f 100644 --- a/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs +++ b/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs @@ -6,6 +6,8 @@ using Microsoft.Build.Tasks; using Microsoft.Build.Shared; using Xunit; +using Shouldly; +using System; namespace Microsoft.Build.UnitTests { @@ -16,35 +18,64 @@ sealed public class ResGenDependencies_Tests public void DirtyCleanScenario(bool useMSBuildResXReader) { - ResGenDependencies cache = new ResGenDependencies(); - + ResGenDependencies cache = new(); string resx = CreateSampleResx(); string stateFile = FileUtilities.GetTemporaryFile(); try { // A newly created cache is not dirty. - Assert.False(cache.IsDirty); + cache.IsDirty.ShouldBeFalse(); + + ResGenDependencies.PortableLibraryFile libFile = new("otherFileName"); + libFile.outputFiles = new string[] { "first", "second" }; + libFile.assemblySimpleName = "simpleName"; + libFile.lastModified = DateTime.Now.Subtract(TimeSpan.FromSeconds(10)); + cache.portableLibraries.AddDependencyFile("fileName", libFile); + + // Writing the file to disk should make the cache clean. + cache.SerializeCache(stateFile, /* Log */ null); + cache.IsDirty.ShouldBeFalse(); // Getting a file that wasn't in the cache is a write operation. cache.GetResXFileInfo(resx, useMSBuildResXReader); - Assert.True(cache.IsDirty); + cache.IsDirty.ShouldBeTrue(); - // Writing the file to disk should make the cache clean. + // Add linkedFiles to further test serialization and deserialization. + cache.resXFiles.dependencies.TryGetValue(resx, out DependencyFile file).ShouldBeTrue(); + (file as ResGenDependencies.ResXFile).linkedFiles = new string[] { "third", "fourth" }; + + // Writing the file to disk should make the cache clean again. cache.SerializeCache(stateFile, /* Log */ null); - Assert.False(cache.IsDirty); + cache.IsDirty.ShouldBeFalse(); // Deserialize from disk. Result should not be dirty. - cache = ResGenDependencies.DeserializeCache(stateFile, true, /* Log */ null); - Assert.False(cache.IsDirty); + ResGenDependencies cache2 = ResGenDependencies.DeserializeCache(stateFile, true, /* Log */ null); + cache2.IsDirty.ShouldBeFalse(); + + // Validate that serialization worked + ResGenDependencies.PortableLibraryFile portableLibrary = cache.portableLibraries.GetDependencyFile("fileName") as ResGenDependencies.PortableLibraryFile; + ResGenDependencies.PortableLibraryFile portableLibrary2 = cache2.portableLibraries.GetDependencyFile("fileName") as ResGenDependencies.PortableLibraryFile; + portableLibrary2.filename.ShouldBe(portableLibrary.filename); + portableLibrary2.exists.ShouldBe(portableLibrary.exists); + portableLibrary2.assemblySimpleName.ShouldBe(portableLibrary.assemblySimpleName); + portableLibrary2.lastModified.ShouldBe(portableLibrary.lastModified); + portableLibrary2.outputFiles.Length.ShouldBe(portableLibrary.outputFiles.Length); + portableLibrary2.outputFiles[1].ShouldBe(portableLibrary.outputFiles[1]); + ResGenDependencies.ResXFile resX = cache.resXFiles.GetDependencyFile(resx) as ResGenDependencies.ResXFile; + ResGenDependencies.ResXFile resX2 = cache2.resXFiles.GetDependencyFile(resx) as ResGenDependencies.ResXFile; + resX2.filename.ShouldBe(resX.filename); + resX2.lastModified.ShouldBe(resX.lastModified); + resX2.linkedFiles.Length.ShouldBe(resX.linkedFiles.Length); + resX2.linkedFiles[1].ShouldBe(resX.linkedFiles[1]); // Asking for a file that's in the cache should not dirty the cache. - cache.GetResXFileInfo(resx, useMSBuildResXReader); - Assert.False(cache.IsDirty); + cache2.GetResXFileInfo(resx, useMSBuildResXReader); + cache2.IsDirty.ShouldBeFalse(); // Changing UseSourcePath to false should dirty the cache. - cache.UseSourcePath = false; - Assert.True(cache.IsDirty); + cache2.UseSourcePath = false; + cache2.IsDirty.ShouldBeTrue(); } finally { From 708b7d9bdb9879588be78933422beb91dc02a505 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Thu, 15 Apr 2021 15:20:32 -0700 Subject: [PATCH 3/8] Remove extra [Serializable] --- src/Tasks/ResGenDependencies.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index e86e81f4e50..f7bb0c906cb 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -186,7 +186,6 @@ internal static ResGenDependencies DeserializeCache(string stateFile, bool useSo /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - [Serializable] internal sealed class ResXFile : DependencyFile { // Files contained within this resx file. @@ -283,7 +282,6 @@ private static string[] GetLinkedFiles(string filename, string baseLinkedFileDir /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - [Serializable] internal sealed class PortableLibraryFile : DependencyFile { internal string[] outputFiles; From fdc980ae91e2806a3cbb959e47892007f1eb14a8 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Fri, 16 Apr 2021 14:34:53 -0700 Subject: [PATCH 4/8] Shift serialization to StateFileBase and extensions --- ...olveAssemblyReferenceCacheSerialization.cs | 181 ++++-------------- .../ResolveAssemblyReference.cs | 8 +- src/Tasks/AssemblyRegistrationCache.cs | 18 +- src/Tasks/Dependencies.cs | 39 +++- src/Tasks/ResGenDependencies.cs | 35 +++- src/Tasks/ResolveComReferenceCache.cs | 16 +- src/Tasks/StateFileBase.cs | 21 +- src/Tasks/SystemState.cs | 99 ++-------- src/Tasks/TaskTranslatorHelpers.cs | 59 ------ 9 files changed, 164 insertions(+), 312 deletions(-) diff --git a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs index 77a9bf0452a..12a999eb34d 100644 --- a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs +++ b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs @@ -1,6 +1,6 @@ using System; +using System.Collections.Generic; using System.IO; -using System.Reflection; using System.Runtime.Versioning; using Microsoft.Build.Shared; using Microsoft.Build.Tasks; @@ -42,79 +42,19 @@ public void RoundTripEmptyState() { SystemState systemState = new(); - systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); + systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper); - var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); + var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState)); deserialized.ShouldNotBeNull(); } - [Fact] - public void WrongFileSignature() - { - SystemState systemState = new(); - - for (int i = 0; i < TranslateContractSignature.Length; i++) - { - systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) - { - cacheStream.Seek(i, SeekOrigin.Begin); - cacheStream.WriteByte(0); - cacheStream.Close(); - } - - var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - - deserialized.ShouldBeNull(); - } - } - - [Fact] - public void WrongFileVersion() - { - SystemState systemState = new(); - - systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) - { - cacheStream.Seek(TranslateContractSignature.Length, SeekOrigin.Begin); - cacheStream.WriteByte((byte) (TranslateContractVersion + 1)); - cacheStream.Close(); - } - - var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - - deserialized.ShouldBeNull(); - } - - [Fact] - public void CorrectFileSignature() - { - SystemState systemState = new(); - - for (int i = 0; i < TranslateContractSignature.Length; i++) - { - systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) - { - cacheStream.Seek(i, SeekOrigin.Begin); - cacheStream.WriteByte(TranslateContractSignature[i]); - cacheStream.Close(); - } - - var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - - deserialized.ShouldNotBeNull(); - } - } - [Fact] public void CorrectFileVersion() { SystemState systemState = new(); - systemState.SerializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); + systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper); using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) { cacheStream.Seek(TranslateContractSignature.Length, SeekOrigin.Begin); @@ -122,98 +62,43 @@ public void CorrectFileVersion() cacheStream.Close(); } - var deserialized = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); + var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState)); deserialized.ShouldNotBeNull(); } [Fact] - public void VerifySampleStateDeserialization() + public void ValidateSerializationAndDeserialization() { - // This test might also fail when binary format is modified. - // Any change in SystemState and child class ITranslatable implementation will most probably make this fail. - // To fix it, file referred by 'sampleName' needs to be recaptured and constant bellow modified to reflect - // the content of that cache. - // This sample was captured by compiling https://github.com/dotnet/roslyn/commit/f8107de2a94a01e96ac3d7c1f225acbb61e18830 - const string sampleName = "Microsoft.VisualStudio.LanguageServices.Implementation.csprojAssemblyReference.cache"; - const string expectedAssemblyPath = @"C:\Users\rokon\.nuget\packages\microsoft.visualstudio.codeanalysis.sdk.ui\15.8.27812-alpha\lib\net46\Microsoft.VisualStudio.CodeAnalysis.Sdk.UI.dll"; - const long expectedAssemblyLastWriteTimeTicks = 636644382480000000; - const string expectedAssemblyName = "Microsoft.VisualStudio.CodeAnalysis.Sdk.UI, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"; - const string expectedFrameworkName = ".NETFramework,Version=v4.5"; - var expectedDependencies = new[] - { - "mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", - "System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", - "System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.CodeAnalysis, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.DeveloperTools, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", - "Microsoft.VisualStudio.Shell.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.CodeAnalysis.Sdk, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.Build.Framework, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Text.Logic, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Text.UI, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Text.Data, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Text.UI.Wpf, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.ComponentModelHost, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.VSHelp, Version=7.0.3300.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Shell.Interop.11.0, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.VCProjectEngine, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Shell.15.0, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.OLE.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "System.Xml, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", - "Microsoft.VisualStudio.TextManager.Interop, Version=7.1.40304.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "System.Core, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089", - "Microsoft.VisualStudio.VirtualTreeGrid, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Shell.Interop.8.0, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - "Microsoft.VisualStudio.Editor, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a", - }; - - - CopyResourceSampleFileIntoRarCacheFile($@"AssemblyDependency\CacheFileSamples\{sampleName}"); - - var deserializedByTranslator = SystemState.DeserializeCacheByTranslator(_rarCacheFile, _taskLoggingHelper); - deserializedByTranslator.ShouldNotBeNull(); - - deserializedByTranslator.SetGetLastWriteTime(path => + Dictionary cache = new() { + { "path1", new SystemState.FileState(DateTime.Now) }, + { "path2", new SystemState.FileState(DateTime.Now) { Assembly = new AssemblyNameExtension("hi") } }, + { "dllName", new SystemState.FileState(DateTime.Now.AddSeconds(-10)) { + Assembly = null, + RuntimeVersion = "v4.0.30319", + FrameworkNameAttribute = new FrameworkName(".NETFramework", Version.Parse("4.7.2"), "Profile"), + scatterFiles = new string[] { "first", "second" } } } }; + SystemState sysState = new(); + sysState.instanceLocalFileStateCache = cache; + SystemState sysState2 = null; + using (TestEnvironment env = TestEnvironment.Create()) { - if (path != expectedAssemblyPath) - throw new InvalidOperationException("Unexpected file name for this test case"); - - return new DateTime(expectedAssemblyLastWriteTimeTicks, DateTimeKind.Utc); - }); - - GetAssemblyName getAssemblyName = deserializedByTranslator.CacheDelegate((GetAssemblyName)null); - GetAssemblyMetadata getAssemblyMetadata = deserializedByTranslator.CacheDelegate((GetAssemblyMetadata)null); - - var assemblyName = getAssemblyName(expectedAssemblyPath); - getAssemblyMetadata(expectedAssemblyPath, null, - out AssemblyNameExtension[] dependencies, - out string[] scatterFiles, - out FrameworkName frameworkNameAttribute); - - - assemblyName.ShouldNotBeNull(); - assemblyName.ShouldBe(new AssemblyNameExtension(expectedAssemblyName, false)); - scatterFiles.ShouldBeEmpty(); - frameworkNameAttribute.ShouldBe(new FrameworkName(expectedFrameworkName)); - dependencies.ShouldNotBeNull(); - expectedDependencies.ShouldBe(expectedDependencies, ignoreOrder: true); - } - - private void CopyResourceSampleFileIntoRarCacheFile(string name) - { - Assembly asm = this.GetType().Assembly; - var resource = string.Format($"{asm.GetName().Name}.{name.Replace("\\", ".")}"); - using Stream resourceStream = asm.GetManifestResourceStream(resource); - if (resourceStream == null) - throw new InvalidOperationException($"Resource '{resource}' has not been found."); - - using FileStream rarCacheFile = new FileStream(_rarCacheFile, FileMode.CreateNew); + TransientTestFile file = env.CreateFile(); + sysState.SerializeCache(file.Path, null); + sysState2 = SystemState.DeserializeCache(file.Path, null, typeof(SystemState)) as SystemState; + } - resourceStream.CopyTo(rarCacheFile); + Dictionary cache2 = sysState2.instanceLocalFileStateCache; + cache2.Count.ShouldBe(cache.Count); + cache2["path2"].Assembly.Name.ShouldBe(cache["path2"].Assembly.Name); + SystemState.FileState dll = cache["dllName"]; + SystemState.FileState dll2 = cache2["dllName"]; + dll2.Assembly.ShouldBe(dll.Assembly); + dll2.FrameworkNameAttribute.FullName.ShouldBe(dll.FrameworkNameAttribute.FullName); + dll2.LastModified.ShouldBe(dll.LastModified); + dll2.RuntimeVersion.ShouldBe(dll.RuntimeVersion); + dll2.scatterFiles.Length.ShouldBe(dll.scatterFiles.Length); + dll2.scatterFiles[1].ShouldBe(dll.scatterFiles[1]); } } } diff --git a/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs b/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs index f0062776ba5..afe90732c04 100644 --- a/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs +++ b/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs @@ -1885,12 +1885,12 @@ private void LogConflict(Reference reference, string fusionName, StringBuilder l /// internal void ReadStateFile(FileExists fileExists) { - _cache = SystemState.DeserializeCacheByTranslator(_stateFile, Log); + _cache = SystemState.DeserializeCache(_stateFile, Log, typeof(SystemState)) as SystemState; // Construct the cache only if we can't find any caches. if (_cache == null && AssemblyInformationCachePaths != null && AssemblyInformationCachePaths.Length > 0) { - _cache = SystemState.DeserializePrecomputedCachesByTranslator(AssemblyInformationCachePaths, Log, fileExists); + _cache = SystemState.DeserializePrecomputedCaches(AssemblyInformationCachePaths, Log, fileExists); } if (_cache == null) @@ -1906,11 +1906,11 @@ internal void WriteStateFile() { if (!String.IsNullOrEmpty(AssemblyInformationCacheOutputPath)) { - _cache.SerializePrecomputedCacheByTranslator(AssemblyInformationCacheOutputPath, Log); + _cache.SerializePrecomputedCache(AssemblyInformationCacheOutputPath, Log); } else if (!String.IsNullOrEmpty(_stateFile) && _cache.IsDirty) { - _cache.SerializeCacheByTranslator(_stateFile, Log); + _cache.SerializeCache(_stateFile, Log); } } #endregion diff --git a/src/Tasks/AssemblyRegistrationCache.cs b/src/Tasks/AssemblyRegistrationCache.cs index eb1f0eda8c3..1c34bd4abd4 100644 --- a/src/Tasks/AssemblyRegistrationCache.cs +++ b/src/Tasks/AssemblyRegistrationCache.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using Microsoft.Build.BackEnd; using Microsoft.Build.Shared; namespace Microsoft.Build.Tasks @@ -9,7 +10,7 @@ namespace Microsoft.Build.Tasks /// /// This class is a caching mechanism for the Register/UnregisterAssembly task to keep track of registered assemblies to clean up /// - internal sealed class AssemblyRegistrationCache : StateFileBase + internal sealed class AssemblyRegistrationCache : StateFileBase, ITranslatable { /// /// The list of registered assembly files. @@ -51,5 +52,20 @@ internal void GetEntry(int index, out string assemblyPath, out string typeLibrar assemblyPath = _assemblies[index]; typeLibraryPath = _typeLibraries[index]; } + + public AssemblyRegistrationCache(ITranslator translator) + { + Translate(translator); + } + + public AssemblyRegistrationCache() { } + + public override void Translate(ITranslator translator) + { + ErrorUtilities.VerifyThrowArgumentNull(translator, nameof(translator)); + translator.Translate(ref _assemblies); + translator.Translate(ref _typeLibraries); + translator.Translate(ref _serializedVersion); + } } } diff --git a/src/Tasks/Dependencies.cs b/src/Tasks/Dependencies.cs index aac1fc0b13b..45cbc87d3c2 100644 --- a/src/Tasks/Dependencies.cs +++ b/src/Tasks/Dependencies.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Build.BackEnd; +using System; using System.Collections.Generic; namespace Microsoft.Build.Tasks @@ -11,17 +13,50 @@ namespace Microsoft.Build.Tasks internal class Dependencies { /// - /// Hashtable of other dependency files. + /// Dictionary of other dependency files. /// Key is filename and value is DependencyFile. /// internal Dictionary dependencies = new(); + internal Dependencies() { } + + internal Dependencies(ITranslator translator, Type t) + { + Translate(translator, t); + } + + public void Translate(ITranslator translator, Type t) + { + translator.TranslateDictionary(ref dependencies, (ITranslator translator, ref DependencyFile dependency) => + { + if (t == typeof(ResGenDependencies.ResXFile)) + { + ResGenDependencies.ResXFile resx = dependency as ResGenDependencies.ResXFile; + resx ??= new(); + resx.Translate(translator); + dependency = resx; + } + else if (t == typeof(ResGenDependencies.PortableLibraryFile)) + { + ResGenDependencies.PortableLibraryFile lib = dependency as ResGenDependencies.PortableLibraryFile; + lib ??= new(); + lib.Translate(translator); + dependency = lib; + } + + dependency ??= new(); + translator.Translate(ref dependency.filename); + translator.Translate(ref dependency.lastModified); + translator.Translate(ref dependency.exists); + }); + } + /// /// Look up a dependency file. Return null if it isn't there. /// /// /// - internal DependencyFile GetDependencyFile(string filename) + internal DependencyFile GetDependencyFile(string filename) { dependencies.TryGetValue(filename, out DependencyFile file); return file; diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index f7bb0c906cb..0962a995b48 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -8,7 +8,7 @@ using System.IO; using System.Resources; using System.Xml; - +using Microsoft.Build.BackEnd; using Microsoft.Build.Shared; using Microsoft.Build.Shared.FileSystem; using Microsoft.Build.Tasks.ResourceHandling; @@ -22,7 +22,7 @@ namespace Microsoft.Build.Tasks /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - internal sealed class ResGenDependencies : StateFileBase + internal sealed class ResGenDependencies : StateFileBase, ITranslatable { /// /// The list of resx files. @@ -89,6 +89,21 @@ internal bool UseSourcePath } } + public ResGenDependencies() { } + + public ResGenDependencies(ITranslator translator) + { + Translate(translator); + } + + public override void Translate(ITranslator translator) + { + resXFiles.Translate(translator, typeof(ResXFile)); + portableLibraries.Translate(translator, typeof(PortableLibraryFile)); + translator.Translate(ref baseLinkedFileDirectory); + translator.Translate(ref _serializedVersion); + } + internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) { // First, try to retrieve the resx information from our hashtable. @@ -186,7 +201,7 @@ internal static ResGenDependencies DeserializeCache(string stateFile, bool useSo /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - internal sealed class ResXFile : DependencyFile + internal sealed class ResXFile : DependencyFile, ITranslatable { // Files contained within this resx file. internal string[] linkedFiles; @@ -210,6 +225,11 @@ internal ResXFile() { } + public void Translate(ITranslator translator) + { + translator.Translate(ref linkedFiles); + } + /// /// Given a .RESX file, returns all the linked files that are referenced within that .RESX. /// @@ -282,7 +302,7 @@ private static string[] GetLinkedFiles(string filename, string baseLinkedFileDir /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - internal sealed class PortableLibraryFile : DependencyFile + internal sealed class PortableLibraryFile : DependencyFile, ITranslatable { internal string[] outputFiles; internal string neutralResourceLanguage; @@ -292,6 +312,13 @@ internal PortableLibraryFile() { } + public void Translate(ITranslator translator) + { + translator.Translate(ref assemblySimpleName); + translator.Translate(ref outputFiles); + translator.Translate(ref neutralResourceLanguage); + } + internal PortableLibraryFile(string filename) : base(filename) { diff --git a/src/Tasks/ResolveComReferenceCache.cs b/src/Tasks/ResolveComReferenceCache.cs index 87fa0210464..aae1745bd55 100644 --- a/src/Tasks/ResolveComReferenceCache.cs +++ b/src/Tasks/ResolveComReferenceCache.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Microsoft.Build.BackEnd; using Microsoft.Build.Shared; namespace Microsoft.Build.Tasks @@ -16,7 +17,7 @@ namespace Microsoft.Build.Tasks /// /// This is an on-disk serialization format, don't change field names or types or use readonly. /// - internal sealed class ResolveComReferenceCache : StateFileBase + internal sealed class ResolveComReferenceCache : StateFileBase, ITranslatable { /// /// Component timestamps. @@ -87,5 +88,18 @@ internal bool ToolPathsMatchCachePaths(string tlbImpPath, string axImpPath) } } } + + public ResolveComReferenceCache(ITranslator translator) + { + Translate(translator); + } + + public override void Translate(ITranslator translator) + { + translator.Translate(ref axImpLocation); + translator.Translate(ref tlbImpLocation); + translator.TranslateDictionary(ref componentTimestamps, StringComparer.Ordinal); + translator.Translate(ref _serializedVersion); + } } } diff --git a/src/Tasks/StateFileBase.cs b/src/Tasks/StateFileBase.cs index 01ec834fa4d..023b7de42d3 100644 --- a/src/Tasks/StateFileBase.cs +++ b/src/Tasks/StateFileBase.cs @@ -13,7 +13,7 @@ namespace Microsoft.Build.Tasks /// /// Base class for task state files. /// - internal class StateFileBase + internal abstract class StateFileBase { // Current version for serialization. This should be changed when breaking changes // are made to this class. @@ -21,10 +21,10 @@ internal class StateFileBase // Version 4/5 - VS2017.7: // Unify .NET Core + Full Framework. Custom serialization on some types that are no // longer [Serializable]. - private const byte CurrentSerializationVersion = 5; + private const byte CurrentSerializationVersion = 6; // Version this instance is serialized with. - private byte _serializedVersion = CurrentSerializationVersion; + protected byte _serializedVersion = CurrentSerializationVersion; /// /// Writes the contents of this object out to the specified file. @@ -43,8 +43,7 @@ internal virtual void SerializeCache(string stateFile, TaskLoggingHelper log) using (var s = new FileStream(stateFile, FileMode.CreateNew)) { var translator = BinaryTranslator.GetWriteTranslator(s); - StateFileBase thisCopy = this; - translator.Translate(ref thisCopy, thisCopy.GetType()); + Translate(translator); } } } @@ -58,6 +57,8 @@ internal virtual void SerializeCache(string stateFile, TaskLoggingHelper log) } } + public abstract void Translate(ITranslator translator); + /// /// Reads the specified file from disk into a StateFileBase derived object. /// @@ -73,7 +74,15 @@ internal static StateFileBase DeserializeCache(string stateFile, TaskLoggingHelp using (FileStream s = new FileStream(stateFile, FileMode.Open)) { var translator = BinaryTranslator.GetReadTranslator(s, buffer: null); - translator.Translate(ref retVal, requiredReturnType); + var constructors = requiredReturnType.GetConstructors(); + foreach (var constructor in constructors) + { + var parameters = constructor.GetParameters(); + if (parameters.Length == 1 && parameters[0].ParameterType == typeof(ITranslator)) + { + retVal = constructor.Invoke(new object[] { translator }) as StateFileBase; + } + } // If retVal is still null or the version is wrong, log a message not a warning. This could be a valid cache with the wrong version preventing correct deserialization. // For the latter case, internals may be unexpectedly null. diff --git a/src/Tasks/SystemState.cs b/src/Tasks/SystemState.cs index d121def739c..7abcbb0a614 100644 --- a/src/Tasks/SystemState.cs +++ b/src/Tasks/SystemState.cs @@ -12,7 +12,6 @@ using Microsoft.Build.BackEnd; using Microsoft.Build.Framework; using Microsoft.Build.Shared; -using Microsoft.Build.Shared.FileSystem; using Microsoft.Build.Tasks.AssemblyDependency; using Microsoft.Build.Utilities; @@ -23,9 +22,6 @@ namespace Microsoft.Build.Tasks /// internal sealed class SystemState : StateFileBase, ITranslatable { - private static readonly byte[] TranslateContractSignature = { (byte) 'M', (byte) 'B', (byte) 'R', (byte) 'S', (byte) 'C'}; // Microsoft Build RAR State Cache - private static readonly byte TranslateContractVersion = 0x01; - /// /// Cache at the SystemState instance level. Has the same contents as . /// It acts as a flag to enforce that an entry has been checked for staleness only once. @@ -219,10 +215,15 @@ internal FrameworkName FrameworkNameAttribute /// /// Construct. /// - internal SystemState() + public SystemState() { } + public SystemState(ITranslator translator) + { + Translate(translator); + } + /// /// Set the target framework paths. /// This is used to optimize IO in the case of files requested from one @@ -237,88 +238,11 @@ AssemblyTableInfo[] installedAssemblyTableInfos redistList = RedistList.GetRedistList(installedAssemblyTableInfos); } - /// - /// Writes the contents of this object out to the specified file. - /// TODO: once all derived classes from StateFileBase adopt new serialization, we shall consider to mode this into base class - /// - internal void SerializeCacheByTranslator(string stateFile, TaskLoggingHelper log) - { - try - { - if (!string.IsNullOrEmpty(stateFile)) - { - if (FileSystems.Default.FileExists(stateFile)) - { - File.Delete(stateFile); - } - - using var s = new FileStream(stateFile, FileMode.CreateNew); - var translator = BinaryTranslator.GetWriteTranslator(s); - - // write file signature - translator.Writer.Write(TranslateContractSignature); - translator.Writer.Write(TranslateContractVersion); - - Translate(translator); - isDirty = false; - } - } - catch (Exception e) when (!ExceptionHandling.NotExpectedSerializationException(e)) - { - // Not being able to serialize the cache is not an error, but we let the user know anyway. - // Don't want to hold up processing just because we couldn't read the file. - log.LogWarningWithCodeFromResources("General.CouldNotWriteStateFile", stateFile, e.Message); - } - } - - /// - /// Read the contents of this object out to the specified file. - /// TODO: once all classes derived from StateFileBase adopt the new serialization, we should consider moving this into the base class - /// - internal static SystemState DeserializeCacheByTranslator(string stateFile, TaskLoggingHelper log) - { - // First, we read the cache from disk if one exists, or if one does not exist, we create one. - try - { - if (!string.IsNullOrEmpty(stateFile) && FileSystems.Default.FileExists(stateFile)) - { - using FileStream s = new FileStream(stateFile, FileMode.Open); - var translator = BinaryTranslator.GetReadTranslator(s, buffer:null); // TODO: shared buffering? - - // verify file signature - var contractSignature = translator.Reader.ReadBytes(TranslateContractSignature.Length); - var contractVersion = translator.Reader.ReadByte(); - - if (!contractSignature.SequenceEqual(TranslateContractSignature) || contractVersion != TranslateContractVersion) - { - log.LogMessageFromResources("General.CouldNotReadStateFileMessage", stateFile, log.FormatResourceString("General.IncompatibleStateFileType")); - return null; - } - - SystemState systemState = new SystemState(); - systemState.Translate(translator); - systemState.isDirty = false; - - return systemState; - } - } - catch (Exception e) when (!ExceptionHandling.IsCriticalException(e)) - { - // The deserialization process seems like it can throw just about - // any exception imaginable. Catch them all here. - // Not being able to deserialize the cache is not an error, but we let the user know anyway. - // Don't want to hold up processing just because we couldn't read the file. - log.LogMessageFromResources("General.CouldNotReadStateFileMessage", stateFile, e.Message); - } - - return null; - } - /// /// Reads/writes this class. /// Used for serialization and deserialization of this class persistent cache. /// - public void Translate(ITranslator translator) + public override void Translate(ITranslator translator) { if (instanceLocalFileStateCache is null) throw new NullReferenceException(nameof(instanceLocalFileStateCache)); @@ -327,6 +251,7 @@ public void Translate(ITranslator translator) ref instanceLocalFileStateCache, StringComparer.OrdinalIgnoreCase, (ITranslator t) => new FileState(t)); + IsDirty = false; } /// @@ -603,7 +528,7 @@ out fileState.frameworkName /// How to log /// Whether a file exists /// A cache representing key aspects of file states. - internal static SystemState DeserializePrecomputedCachesByTranslator(ITaskItem[] stateFiles, TaskLoggingHelper log, FileExists fileExists) + internal static SystemState DeserializePrecomputedCaches(ITaskItem[] stateFiles, TaskLoggingHelper log, FileExists fileExists) { SystemState retVal = new SystemState(); retVal.isDirty = stateFiles.Length > 0; @@ -612,7 +537,7 @@ internal static SystemState DeserializePrecomputedCachesByTranslator(ITaskItem[] foreach (ITaskItem stateFile in stateFiles) { // Verify that it's a real stateFile. Log message but do not error if not. - SystemState sysState = DeserializeCacheByTranslator(stateFile.ToString(), log); + SystemState sysState = DeserializeCache(stateFile.ToString(), log, typeof(SystemState)) as SystemState; if (sysState == null) { continue; @@ -642,7 +567,7 @@ internal static SystemState DeserializePrecomputedCachesByTranslator(ITaskItem[] /// /// Path to which to write the precomputed cache /// How to log - internal void SerializePrecomputedCacheByTranslator(string stateFile, TaskLoggingHelper log) + internal void SerializePrecomputedCache(string stateFile, TaskLoggingHelper log) { // Save a copy of instanceLocalFileStateCache so we can restore it later. SerializeCacheByTranslator serializes // instanceLocalFileStateCache by default, so change that to the relativized form, then change it back. @@ -655,7 +580,7 @@ internal void SerializePrecomputedCacheByTranslator(string stateFile, TaskLoggin { log.LogWarningWithCodeFromResources("General.StateFileAlreadyPresent", stateFile); } - SerializeCacheByTranslator(stateFile, log); + SerializeCache(stateFile, log); } finally { diff --git a/src/Tasks/TaskTranslatorHelpers.cs b/src/Tasks/TaskTranslatorHelpers.cs index c5bb030a4e4..e0d7bb4baa0 100644 --- a/src/Tasks/TaskTranslatorHelpers.cs +++ b/src/Tasks/TaskTranslatorHelpers.cs @@ -65,64 +65,5 @@ public static void TranslateDictionary(this ITranslator translator, ref Dictiona } } } - - public static void Translate(this ITranslator translator, ref Dependencies dependencies, Type t) - { - translator.TranslateDictionary(ref dependencies.dependencies, (ITranslator translator, ref DependencyFile dependency) => { - if (t == typeof(ResGenDependencies.ResXFile)) - { - ResGenDependencies.ResXFile resx = dependency as ResGenDependencies.ResXFile; - resx ??= new(); - translator.Translate(ref resx.linkedFiles); - dependency = resx; - } - else if (t == typeof(ResGenDependencies.PortableLibraryFile)) - { - ResGenDependencies.PortableLibraryFile lib = dependency as ResGenDependencies.PortableLibraryFile; - lib ??= new(); - translator.Translate(ref lib.assemblySimpleName); - translator.Translate(ref lib.outputFiles); - translator.Translate(ref lib.neutralResourceLanguage); - dependency = lib; - } - - dependency ??= new(); - translator.Translate(ref dependency.filename); - translator.Translate(ref dependency.lastModified); - translator.Translate(ref dependency.exists); - }); - } - - public static void Translate(this ITranslator translator, ref StateFileBase stateFile, Type t) - { - if (t == typeof(ResGenDependencies)) - { - ResGenDependencies rgd = stateFile as ResGenDependencies; - rgd ??= new(); - translator.Translate(ref rgd.resXFiles, typeof(ResGenDependencies.ResXFile)); - translator.Translate(ref rgd.portableLibraries, typeof(ResGenDependencies.PortableLibraryFile)); - translator.Translate(ref rgd.baseLinkedFileDirectory); - stateFile = rgd; - } -#if NETFRAMEWORK - else if (t == typeof(ResolveComReferenceCache)) - { - ResolveComReferenceCache rcrc = stateFile as ResolveComReferenceCache; - rcrc ??= new(string.Empty, string.Empty); - translator.Translate(ref rcrc.axImpLocation); - translator.Translate(ref rcrc.tlbImpLocation); - translator.TranslateDictionary(ref rcrc.componentTimestamps, StringComparer.Ordinal); - stateFile = rcrc; - } - else if (t == typeof(AssemblyRegistrationCache)) - { - AssemblyRegistrationCache arc = stateFile as AssemblyRegistrationCache; - arc ??= new(); - translator.Translate(ref arc._assemblies); - translator.Translate(ref arc._typeLibraries); - stateFile = arc; - } -#endif - } } } From fc25eb772d31c9c92a389404623c4f528a34614c Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Fri, 16 Apr 2021 15:09:52 -0700 Subject: [PATCH 5/8] Add copyright header and remove fileExists --- .../ResolveAssemblyReferenceCacheSerialization.cs | 2 ++ src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs | 2 +- src/Tasks/SystemState.cs | 9 +-------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs index 12a999eb34d..7e29179a750 100644 --- a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs +++ b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs @@ -1,3 +1,5 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; using System.Collections.Generic; using System.IO; diff --git a/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs b/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs index afe90732c04..4b47053fb6e 100644 --- a/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs +++ b/src/Tasks/AssemblyDependency/ResolveAssemblyReference.cs @@ -2147,7 +2147,7 @@ ReadMachineTypeFromPEHeader readMachineTypeFromPEHeader // Cache delegates. getAssemblyName = _cache.CacheDelegate(getAssemblyName); getAssemblyMetadata = _cache.CacheDelegate(getAssemblyMetadata); - fileExists = _cache.CacheDelegate(fileExists); + fileExists = _cache.CacheDelegate(); directoryExists = _cache.CacheDelegate(directoryExists); getDirectories = _cache.CacheDelegate(getDirectories); getRuntimeVersion = _cache.CacheDelegate(getRuntimeVersion); diff --git a/src/Tasks/SystemState.cs b/src/Tasks/SystemState.cs index 7abcbb0a614..71b8f873f2d 100644 --- a/src/Tasks/SystemState.cs +++ b/src/Tasks/SystemState.cs @@ -84,11 +84,6 @@ internal sealed class SystemState : StateFileBase, ITranslatable /// private GetAssemblyMetadata getAssemblyMetadata; - /// - /// Cached delegate. - /// - private FileExists fileExists; - /// /// Cached delegate. /// @@ -298,11 +293,9 @@ internal GetAssemblyMetadata CacheDelegate(GetAssemblyMetadata getAssemblyMetada /// /// Cache the results of a FileExists delegate. /// - /// The delegate. /// Cached version of the delegate. - internal FileExists CacheDelegate(FileExists fileExistsValue) + internal FileExists CacheDelegate() { - fileExists = fileExistsValue; return FileExists; } From 76b99233d6a1ff8678af805d30f5b935eee95aa3 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Tue, 20 Apr 2021 10:45:49 -0700 Subject: [PATCH 6/8] Partially implement suggestions --- src/Shared/BinaryTranslator.cs | 35 +++++++++++++++++++ src/Shared/ITranslator.cs | 2 ++ ...olveAssemblyReferenceCacheSerialization.cs | 1 + src/Tasks/ResGenDependencies.cs | 2 +- src/Tasks/SystemState.cs | 3 ++ src/Tasks/TaskTranslatorHelpers.cs | 31 ---------------- 6 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/Shared/BinaryTranslator.cs b/src/Shared/BinaryTranslator.cs index 6c2b6337393..875f276b48e 100644 --- a/src/Shared/BinaryTranslator.cs +++ b/src/Shared/BinaryTranslator.cs @@ -1254,6 +1254,41 @@ public void TranslateDictionary(ref Dictionary dictionary, IEquali } } + /// + /// Translates a dictionary of { string, T } for dictionaries with public parameterless constructors. + /// + /// The dictionary to be translated. + /// Key comparer + public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer) + { + int count = 0; + if (Mode == TranslationDirection.ReadFromStream) + { + dictionary = new(comparer); + Translate(ref count); + string key = string.Empty; + DateTime val = DateTime.MinValue; + for (int i = 0; i < count; i++) + { + Translate(ref key); + Translate(ref val); + dictionary.Add(key, val); + } + } + else + { + count = dictionary.Count; + Translate(ref count); + foreach (KeyValuePair kvp in dictionary) + { + string key = kvp.Key; + DateTime val = kvp.Value; + Translate(ref key); + Translate(ref val); + } + } + } + /// /// Writes out the boolean which says if this object is null or not. /// diff --git a/src/Shared/ITranslator.cs b/src/Shared/ITranslator.cs index b1acb85ec2f..55e56c24746 100644 --- a/src/Shared/ITranslator.cs +++ b/src/Shared/ITranslator.cs @@ -301,6 +301,8 @@ void TranslateArray(ref T[] array) void TranslateDictionary(ref IDictionary dictionary, NodePacketCollectionCreator> collectionCreator); + public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer); + void TranslateDictionary(ref IDictionary dictionary, ObjectTranslator keyTranslator, ObjectTranslator valueTranslator, NodePacketCollectionCreator> dictionaryCreator); /// diff --git a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs index 7e29179a750..2f0a70d2204 100644 --- a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs +++ b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. + using System; using System.Collections.Generic; using System.IO; diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index 0962a995b48..af9f6e2ffb9 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -107,7 +107,7 @@ public override void Translate(ITranslator translator) internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) { // First, try to retrieve the resx information from our hashtable. - if (resXFiles.GetDependencyFile(resxFile) is not ResXFile retVal || retVal == null) + if ((resXFiles.GetDependencyFile(resxFile) as ResXFile retVal) is null) { // Ok, the file wasn't there. Add it to our cache and return it to the caller. retVal = AddResxFile(resxFile, useMSBuildResXReader); diff --git a/src/Tasks/SystemState.cs b/src/Tasks/SystemState.cs index 71b8f873f2d..a94833887f0 100644 --- a/src/Tasks/SystemState.cs +++ b/src/Tasks/SystemState.cs @@ -246,6 +246,9 @@ public override void Translate(ITranslator translator) ref instanceLocalFileStateCache, StringComparer.OrdinalIgnoreCase, (ITranslator t) => new FileState(t)); + + // IsDirty should be false for either direction. Either this cache was brought + // up-to-date with the on-disk cache or vice versa. Either way, they agree. IsDirty = false; } diff --git a/src/Tasks/TaskTranslatorHelpers.cs b/src/Tasks/TaskTranslatorHelpers.cs index e0d7bb4baa0..7db48cf61ae 100644 --- a/src/Tasks/TaskTranslatorHelpers.cs +++ b/src/Tasks/TaskTranslatorHelpers.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Runtime.Versioning; using Microsoft.Build.BackEnd; @@ -35,35 +34,5 @@ public static void Translate(this ITranslator translator, ref FrameworkName fram frameworkName = new FrameworkName(identifier, version, profile); } } - - public static void TranslateDictionary(this ITranslator translator, ref Dictionary dict, StringComparer comparer) - { - int count = 0; - if (translator.Mode == TranslationDirection.ReadFromStream) - { - dict = new Dictionary(comparer); - translator.Translate(ref count); - string key = string.Empty; - DateTime val = DateTime.Now; - for (int i = 0; i < count; i++) - { - translator.Translate(ref key); - translator.Translate(ref val); - dict.Add(key, val); - } - } - else - { - count = dict.Count; - translator.Translate(ref count); - foreach (KeyValuePair kvp in dict) - { - string key = kvp.Key; - DateTime val = kvp.Value; - translator.Translate(ref key); - translator.Translate(ref val); - } - } - } } } From 858a99b5a9a9d0e1dfdaf6305d64d7e10d14fa97 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Tue, 20 Apr 2021 13:32:17 -0700 Subject: [PATCH 7/8] Remove Dependencies --- src/Shared/BinaryTranslator.cs | 58 ++++++------ src/Shared/ITranslator.cs | 2 +- .../ResGenDependencies_Tests.cs | 14 +-- src/Tasks/Dependencies.cs | 89 ------------------- src/Tasks/Microsoft.Build.Tasks.csproj | 3 +- src/Tasks/ResGenDependencies.cs | 43 ++++++--- 6 files changed, 68 insertions(+), 141 deletions(-) delete mode 100644 src/Tasks/Dependencies.cs diff --git a/src/Shared/BinaryTranslator.cs b/src/Shared/BinaryTranslator.cs index 875f276b48e..b200004c15d 100644 --- a/src/Shared/BinaryTranslator.cs +++ b/src/Shared/BinaryTranslator.cs @@ -660,12 +660,27 @@ public void TranslateDictionary(ref Dictionary dictionary, IEquali } } - /// - /// Reads in the boolean which says if this object is null or not. - /// - /// The type of object to test. - /// True if the object should be read, false otherwise. - public bool TranslateNullable(T value) + public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer) + { + int count = 0; + dictionary = new(comparer); + Translate(ref count); + string key = string.Empty; + DateTime val = DateTime.MinValue; + for (int i = 0; i < count; i++) + { + Translate(ref key); + Translate(ref val); + dictionary.Add(key, val); + } + } + + /// + /// Reads in the boolean which says if this object is null or not. + /// + /// The type of object to test. + /// True if the object should be read, false otherwise. + public bool TranslateNullable(T value) { bool haveRef = _reader.ReadBoolean(); return haveRef; @@ -1261,31 +1276,14 @@ public void TranslateDictionary(ref Dictionary dictionary, IEquali /// Key comparer public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer) { - int count = 0; - if (Mode == TranslationDirection.ReadFromStream) - { - dictionary = new(comparer); - Translate(ref count); - string key = string.Empty; - DateTime val = DateTime.MinValue; - for (int i = 0; i < count; i++) - { - Translate(ref key); - Translate(ref val); - dictionary.Add(key, val); - } - } - else + int count = dictionary.Count; + Translate(ref count); + foreach (KeyValuePair kvp in dictionary) { - count = dictionary.Count; - Translate(ref count); - foreach (KeyValuePair kvp in dictionary) - { - string key = kvp.Key; - DateTime val = kvp.Value; - Translate(ref key); - Translate(ref val); - } + string key = kvp.Key; + DateTime val = kvp.Value; + Translate(ref key); + Translate(ref val); } } diff --git a/src/Shared/ITranslator.cs b/src/Shared/ITranslator.cs index 55e56c24746..97ea429de96 100644 --- a/src/Shared/ITranslator.cs +++ b/src/Shared/ITranslator.cs @@ -301,7 +301,7 @@ void TranslateArray(ref T[] array) void TranslateDictionary(ref IDictionary dictionary, NodePacketCollectionCreator> collectionCreator); - public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer); + void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer); void TranslateDictionary(ref IDictionary dictionary, ObjectTranslator keyTranslator, ObjectTranslator valueTranslator, NodePacketCollectionCreator> dictionaryCreator); diff --git a/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs b/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs index bb4ca7de48f..fa4c5af5675 100644 --- a/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs +++ b/src/Tasks.UnitTests/ResourceHandling/ResGenDependencies_Tests.cs @@ -31,7 +31,7 @@ public void DirtyCleanScenario(bool useMSBuildResXReader) libFile.outputFiles = new string[] { "first", "second" }; libFile.assemblySimpleName = "simpleName"; libFile.lastModified = DateTime.Now.Subtract(TimeSpan.FromSeconds(10)); - cache.portableLibraries.AddDependencyFile("fileName", libFile); + cache.portableLibraries.Add("fileName", libFile); // Writing the file to disk should make the cache clean. cache.SerializeCache(stateFile, /* Log */ null); @@ -42,8 +42,8 @@ public void DirtyCleanScenario(bool useMSBuildResXReader) cache.IsDirty.ShouldBeTrue(); // Add linkedFiles to further test serialization and deserialization. - cache.resXFiles.dependencies.TryGetValue(resx, out DependencyFile file).ShouldBeTrue(); - (file as ResGenDependencies.ResXFile).linkedFiles = new string[] { "third", "fourth" }; + cache.resXFiles.TryGetValue(resx, out ResGenDependencies.ResXFile file).ShouldBeTrue(); + file.linkedFiles = new string[] { "third", "fourth" }; // Writing the file to disk should make the cache clean again. cache.SerializeCache(stateFile, /* Log */ null); @@ -54,16 +54,16 @@ public void DirtyCleanScenario(bool useMSBuildResXReader) cache2.IsDirty.ShouldBeFalse(); // Validate that serialization worked - ResGenDependencies.PortableLibraryFile portableLibrary = cache.portableLibraries.GetDependencyFile("fileName") as ResGenDependencies.PortableLibraryFile; - ResGenDependencies.PortableLibraryFile portableLibrary2 = cache2.portableLibraries.GetDependencyFile("fileName") as ResGenDependencies.PortableLibraryFile; + cache.portableLibraries.TryGetValue("fileName", out ResGenDependencies.PortableLibraryFile portableLibrary); + cache2.portableLibraries.TryGetValue("fileName", out ResGenDependencies.PortableLibraryFile portableLibrary2); portableLibrary2.filename.ShouldBe(portableLibrary.filename); portableLibrary2.exists.ShouldBe(portableLibrary.exists); portableLibrary2.assemblySimpleName.ShouldBe(portableLibrary.assemblySimpleName); portableLibrary2.lastModified.ShouldBe(portableLibrary.lastModified); portableLibrary2.outputFiles.Length.ShouldBe(portableLibrary.outputFiles.Length); portableLibrary2.outputFiles[1].ShouldBe(portableLibrary.outputFiles[1]); - ResGenDependencies.ResXFile resX = cache.resXFiles.GetDependencyFile(resx) as ResGenDependencies.ResXFile; - ResGenDependencies.ResXFile resX2 = cache2.resXFiles.GetDependencyFile(resx) as ResGenDependencies.ResXFile; + cache.resXFiles.TryGetValue(resx, out ResGenDependencies.ResXFile resX); + cache2.resXFiles.TryGetValue(resx, out ResGenDependencies.ResXFile resX2); resX2.filename.ShouldBe(resX.filename); resX2.lastModified.ShouldBe(resX.lastModified); resX2.linkedFiles.Length.ShouldBe(resX.linkedFiles.Length); diff --git a/src/Tasks/Dependencies.cs b/src/Tasks/Dependencies.cs deleted file mode 100644 index 45cbc87d3c2..00000000000 --- a/src/Tasks/Dependencies.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Microsoft.Build.BackEnd; -using System; -using System.Collections.Generic; - -namespace Microsoft.Build.Tasks -{ - /// - /// Represents a cache of inputs to a compilation-style task. - /// - internal class Dependencies - { - /// - /// Dictionary of other dependency files. - /// Key is filename and value is DependencyFile. - /// - internal Dictionary dependencies = new(); - - internal Dependencies() { } - - internal Dependencies(ITranslator translator, Type t) - { - Translate(translator, t); - } - - public void Translate(ITranslator translator, Type t) - { - translator.TranslateDictionary(ref dependencies, (ITranslator translator, ref DependencyFile dependency) => - { - if (t == typeof(ResGenDependencies.ResXFile)) - { - ResGenDependencies.ResXFile resx = dependency as ResGenDependencies.ResXFile; - resx ??= new(); - resx.Translate(translator); - dependency = resx; - } - else if (t == typeof(ResGenDependencies.PortableLibraryFile)) - { - ResGenDependencies.PortableLibraryFile lib = dependency as ResGenDependencies.PortableLibraryFile; - lib ??= new(); - lib.Translate(translator); - dependency = lib; - } - - dependency ??= new(); - translator.Translate(ref dependency.filename); - translator.Translate(ref dependency.lastModified); - translator.Translate(ref dependency.exists); - }); - } - - /// - /// Look up a dependency file. Return null if it isn't there. - /// - /// - /// - internal DependencyFile GetDependencyFile(string filename) - { - dependencies.TryGetValue(filename, out DependencyFile file); - return file; - } - - /// - /// Add a new dependency file. - /// - internal void AddDependencyFile(string filename, DependencyFile file) - { - dependencies[filename] = file; - } - - /// - /// Remove new dependency file. - /// - internal void RemoveDependencyFile(string filename) - { - dependencies.Remove(filename); - } - - /// - /// Remove all entries from the dependency table. - /// - internal void Clear() - { - dependencies.Clear(); - } - } -} diff --git a/src/Tasks/Microsoft.Build.Tasks.csproj b/src/Tasks/Microsoft.Build.Tasks.csproj index 6e05beb2f75..ff5b4b525d0 100644 --- a/src/Tasks/Microsoft.Build.Tasks.csproj +++ b/src/Tasks/Microsoft.Build.Tasks.csproj @@ -91,7 +91,7 @@ NGen.cs - + PropertyParser.cs True @@ -550,7 +550,6 @@ - diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index af9f6e2ffb9..55f67c4b0f4 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -27,12 +27,12 @@ internal sealed class ResGenDependencies : StateFileBase, ITranslatable /// /// The list of resx files. /// - internal Dependencies resXFiles = new Dependencies(); + internal IDictionary resXFiles = new Dictionary(); /// /// A list of portable libraries and the ResW files they can produce. /// - internal Dependencies portableLibraries = new Dependencies(); + internal IDictionary portableLibraries = new Dictionary(); /// /// A newly-created ResGenDependencies is not dirty. @@ -98,8 +98,22 @@ public ResGenDependencies(ITranslator translator) public override void Translate(ITranslator translator) { - resXFiles.Translate(translator, typeof(ResXFile)); - portableLibraries.Translate(translator, typeof(PortableLibraryFile)); + translator.TranslateDictionary(ref resXFiles, + (ITranslator translator, ref string s) => translator.Translate(ref s), + (ITranslator translator, ref ResXFile resx) => { + ResXFile temp = resx ?? new(); + temp.Translate(translator); + resx = temp; + }, + count => new Dictionary(count)); + translator.TranslateDictionary(ref portableLibraries, + (ITranslator translator, ref string s) => translator.Translate(ref s), + (ITranslator translator, ref PortableLibraryFile portableLibrary) => { + PortableLibraryFile temp = portableLibrary ?? new(); + temp.Translate(translator); + portableLibrary = temp; + }, + count => new Dictionary(count)); translator.Translate(ref baseLinkedFileDirectory); translator.Translate(ref _serializedVersion); } @@ -107,7 +121,7 @@ public override void Translate(ITranslator translator) internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) { // First, try to retrieve the resx information from our hashtable. - if ((resXFiles.GetDependencyFile(resxFile) as ResXFile retVal) is null) + if (!resXFiles.TryGetValue(resxFile, out ResXFile retVal)) { // Ok, the file wasn't there. Add it to our cache and return it to the caller. retVal = AddResxFile(resxFile, useMSBuildResXReader); @@ -118,7 +132,7 @@ internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) // by removing it from the hashtable and readding it. if (retVal.HasFileChanged()) { - resXFiles.RemoveDependencyFile(resxFile); + resXFiles.Remove(resxFile); _isDirty = true; retVal = AddResxFile(resxFile, useMSBuildResXReader); } @@ -133,7 +147,7 @@ private ResXFile AddResxFile(string file, bool useMSBuildResXReader) // to be cracked for contained files. var resxFile = new ResXFile(file, BaseLinkedFileDirectory, useMSBuildResXReader); - resXFiles.AddDependencyFile(file, resxFile); + resXFiles.Add(file, resxFile); _isDirty = true; return resxFile; } @@ -141,13 +155,13 @@ private ResXFile AddResxFile(string file, bool useMSBuildResXReader) internal PortableLibraryFile TryGetPortableLibraryInfo(string libraryPath) { // First, try to retrieve the portable library information from our hashtable. - var retVal = (PortableLibraryFile)portableLibraries.GetDependencyFile(libraryPath); + portableLibraries.TryGetValue(libraryPath, out PortableLibraryFile retVal); // The file is in our cache. Make sure it's up to date. If not, discard // this entry from the cache and rebuild all the state at a later point. if (retVal?.HasFileChanged() == true) { - portableLibraries.RemoveDependencyFile(libraryPath); + portableLibraries.Remove(libraryPath); _isDirty = true; retVal = null; } @@ -157,11 +171,10 @@ internal PortableLibraryFile TryGetPortableLibraryInfo(string libraryPath) internal void UpdatePortableLibrary(PortableLibraryFile library) { - var cached = (PortableLibraryFile)portableLibraries.GetDependencyFile(library.FileName); - if (cached == null || !library.Equals(cached)) + if (!portableLibraries.TryGetValue(library.FileName, out PortableLibraryFile cached) || !library.Equals(cached)) { // Add a new entry or replace the existing one. - portableLibraries.AddDependencyFile(library.FileName, library); + portableLibraries.Add(library.FileName, library); _isDirty = true; } } @@ -228,6 +241,9 @@ internal ResXFile() public void Translate(ITranslator translator) { translator.Translate(ref linkedFiles); + translator.Translate(ref filename); + translator.Translate(ref lastModified); + translator.Translate(ref exists); } /// @@ -317,6 +333,9 @@ public void Translate(ITranslator translator) translator.Translate(ref assemblySimpleName); translator.Translate(ref outputFiles); translator.Translate(ref neutralResourceLanguage); + translator.Translate(ref filename); + translator.Translate(ref lastModified); + translator.Translate(ref exists); } internal PortableLibraryFile(string filename) From 69ad8b7629596eec593a793eba4f9eedc8d3fef4 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Tue, 27 Apr 2021 17:25:51 -0700 Subject: [PATCH 8/8] Comments. Also centralize version --- src/Shared/BinaryTranslator.cs | 19 ++++++++++---- ...olveAssemblyReferenceCacheSerialization.cs | 26 ++++++++++++++----- src/Tasks/AssemblyRegistrationCache.cs | 1 - src/Tasks/ResGenDependencies.cs | 1 - src/Tasks/ResolveComReferenceCache.cs | 1 - src/Tasks/StateFileBase.cs | 9 ++++--- 6 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/Shared/BinaryTranslator.cs b/src/Shared/BinaryTranslator.cs index b200004c15d..74f46865a14 100644 --- a/src/Shared/BinaryTranslator.cs +++ b/src/Shared/BinaryTranslator.cs @@ -662,9 +662,13 @@ public void TranslateDictionary(ref Dictionary dictionary, IEquali public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer) { - int count = 0; - dictionary = new(comparer); - Translate(ref count); + if (!TranslateNullable(dictionary)) + { + return; + } + + int count = _reader.ReadInt32(); + dictionary = new(count, comparer); string key = string.Empty; DateTime val = DateTime.MinValue; for (int i = 0; i < count; i++) @@ -1270,14 +1274,19 @@ public void TranslateDictionary(ref Dictionary dictionary, IEquali } /// - /// Translates a dictionary of { string, T } for dictionaries with public parameterless constructors. + /// Translates a dictionary of { string, DateTime }. /// /// The dictionary to be translated. /// Key comparer public void TranslateDictionary(ref Dictionary dictionary, StringComparer comparer) { + if (!TranslateNullable(dictionary)) + { + return; + } + int count = dictionary.Count; - Translate(ref count); + _writer.Write(count); foreach (KeyValuePair kvp in dictionary) { string key = kvp.Key; diff --git a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs index 2f0a70d2204..7f55c5bcbd1 100644 --- a/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs +++ b/src/Tasks.UnitTests/AssemblyDependency/ResolveAssemblyReferenceCacheSerialization.cs @@ -15,10 +15,6 @@ namespace Microsoft.Build.UnitTests.ResolveAssemblyReference_Tests { public class ResolveAssemblyReferenceCacheSerialization : IDisposable { - // Maintain this two in sync with the constant in SystemState - private static readonly byte[] TranslateContractSignature = { (byte)'M', (byte)'B', (byte)'R', (byte)'S', (byte)'C' }; // Microsoft Build RAR State Cache - private static readonly byte TranslateContractVersion = 0x01; - private readonly string _rarCacheFile; private readonly TaskLoggingHelper _taskLoggingHelper; @@ -60,8 +56,8 @@ public void CorrectFileVersion() systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper); using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) { - cacheStream.Seek(TranslateContractSignature.Length, SeekOrigin.Begin); - cacheStream.WriteByte(TranslateContractVersion); + cacheStream.Seek(0, SeekOrigin.Begin); + cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion); cacheStream.Close(); } @@ -70,6 +66,24 @@ public void CorrectFileVersion() deserialized.ShouldNotBeNull(); } + [Fact] + public void WrongFileVersion() + { + SystemState systemState = new(); + + systemState.SerializeCache(_rarCacheFile, _taskLoggingHelper); + using (var cacheStream = new FileStream(_rarCacheFile, FileMode.Open, FileAccess.ReadWrite)) + { + cacheStream.Seek(0, SeekOrigin.Begin); + cacheStream.WriteByte(StateFileBase.CurrentSerializationVersion - 1); + cacheStream.Close(); + } + + var deserialized = SystemState.DeserializeCache(_rarCacheFile, _taskLoggingHelper, typeof(SystemState)); + + deserialized.ShouldBeNull(); + } + [Fact] public void ValidateSerializationAndDeserialization() { diff --git a/src/Tasks/AssemblyRegistrationCache.cs b/src/Tasks/AssemblyRegistrationCache.cs index 1c34bd4abd4..9c94ed3aefa 100644 --- a/src/Tasks/AssemblyRegistrationCache.cs +++ b/src/Tasks/AssemblyRegistrationCache.cs @@ -65,7 +65,6 @@ public override void Translate(ITranslator translator) ErrorUtilities.VerifyThrowArgumentNull(translator, nameof(translator)); translator.Translate(ref _assemblies); translator.Translate(ref _typeLibraries); - translator.Translate(ref _serializedVersion); } } } diff --git a/src/Tasks/ResGenDependencies.cs b/src/Tasks/ResGenDependencies.cs index 55f67c4b0f4..4e2d06bafee 100644 --- a/src/Tasks/ResGenDependencies.cs +++ b/src/Tasks/ResGenDependencies.cs @@ -115,7 +115,6 @@ public override void Translate(ITranslator translator) }, count => new Dictionary(count)); translator.Translate(ref baseLinkedFileDirectory); - translator.Translate(ref _serializedVersion); } internal ResXFile GetResXFileInfo(string resxFile, bool useMSBuildResXReader) diff --git a/src/Tasks/ResolveComReferenceCache.cs b/src/Tasks/ResolveComReferenceCache.cs index aae1745bd55..ec73dc72c57 100644 --- a/src/Tasks/ResolveComReferenceCache.cs +++ b/src/Tasks/ResolveComReferenceCache.cs @@ -99,7 +99,6 @@ public override void Translate(ITranslator translator) translator.Translate(ref axImpLocation); translator.Translate(ref tlbImpLocation); translator.TranslateDictionary(ref componentTimestamps, StringComparer.Ordinal); - translator.Translate(ref _serializedVersion); } } } diff --git a/src/Tasks/StateFileBase.cs b/src/Tasks/StateFileBase.cs index 023b7de42d3..f228e84bd07 100644 --- a/src/Tasks/StateFileBase.cs +++ b/src/Tasks/StateFileBase.cs @@ -21,10 +21,10 @@ internal abstract class StateFileBase // Version 4/5 - VS2017.7: // Unify .NET Core + Full Framework. Custom serialization on some types that are no // longer [Serializable]. - private const byte CurrentSerializationVersion = 6; + internal const byte CurrentSerializationVersion = 6; // Version this instance is serialized with. - protected byte _serializedVersion = CurrentSerializationVersion; + private byte _serializedVersion = CurrentSerializationVersion; /// /// Writes the contents of this object out to the specified file. @@ -43,6 +43,7 @@ internal virtual void SerializeCache(string stateFile, TaskLoggingHelper log) using (var s = new FileStream(stateFile, FileMode.CreateNew)) { var translator = BinaryTranslator.GetWriteTranslator(s); + translator.Translate(ref _serializedVersion); Translate(translator); } } @@ -74,6 +75,8 @@ internal static StateFileBase DeserializeCache(string stateFile, TaskLoggingHelp using (FileStream s = new FileStream(stateFile, FileMode.Open)) { var translator = BinaryTranslator.GetReadTranslator(s, buffer: null); + byte version = 0; + translator.Translate(ref version); var constructors = requiredReturnType.GetConstructors(); foreach (var constructor in constructors) { @@ -86,7 +89,7 @@ internal static StateFileBase DeserializeCache(string stateFile, TaskLoggingHelp // If retVal is still null or the version is wrong, log a message not a warning. This could be a valid cache with the wrong version preventing correct deserialization. // For the latter case, internals may be unexpectedly null. - if (retVal == null || retVal._serializedVersion != CurrentSerializationVersion) + if (retVal == null || version != CurrentSerializationVersion) { // When upgrading to Visual Studio 2008 and running the build for the first time the resource cache files are replaced which causes a cast error due // to a new version number on the tasks class. "Unable to cast object of type 'Microsoft.Build.Tasks.SystemState' to type 'Microsoft.Build.Tasks.StateFileBase".