From 0eae09f861acd86ad8980f3beccd4d1673163b15 Mon Sep 17 00:00:00 2001 From: Jakub Chocholowicz Date: Mon, 23 Nov 2020 12:51:07 +0100 Subject: [PATCH 1/5] Downgrade externals - tests for test platform nuget --- scripts/build/TestPlatform.Dependencies.props | 2 +- scripts/verify-nupkgs.ps1 | 2 +- .../nuspec/Microsoft.TestPlatform.nuspec | 1 + .../CodeCoverageAcceptanceTestBase.cs | 32 ++++++ .../CodeCoverageTests.cs | 32 ------ .../TestPlatformNugetPackageTests.cs | 102 ++++++++++++++++++ 6 files changed, 137 insertions(+), 34 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 8212613327..8c5a9d2835 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -31,7 +31,7 @@ 5.0.0 9.0.1 4.7.63 - 16.9.0-preview-4243816 + 16.9.0-preview-4224421 16.9.0-beta.20570.2 16.8.0-beta.20420.2 diff --git a/scripts/verify-nupkgs.ps1 b/scripts/verify-nupkgs.ps1 index 70f48ec763..3b2e95f451 100644 --- a/scripts/verify-nupkgs.ps1 +++ b/scripts/verify-nupkgs.ps1 @@ -15,7 +15,7 @@ function Verify-Nuget-Packages($packageDirectory, $version) $expectedNumOfFiles = @{ "Microsoft.CodeCoverage" = 44; "Microsoft.NET.Test.Sdk" = 18; - "Microsoft.TestPlatform" = 486; + "Microsoft.TestPlatform" = 487; "Microsoft.TestPlatform.Build" = 19; "Microsoft.TestPlatform.CLI" = 353; "Microsoft.TestPlatform.Extensions.TrxLogger" = 33; diff --git a/src/package/nuspec/Microsoft.TestPlatform.nuspec b/src/package/nuspec/Microsoft.TestPlatform.nuspec index 504f900f82..1d2e650fd7 100644 --- a/src/package/nuspec/Microsoft.TestPlatform.nuspec +++ b/src/package/nuspec/Microsoft.TestPlatform.nuspec @@ -267,6 +267,7 @@ + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index 5401266b1c..a05b509d20 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -75,5 +75,37 @@ protected void AssertCoverage(XmlNode node, double expectedCoverage) Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}"); Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}"); } + + protected static string GetCoverageFileNameFromTrx(string trxFilePath, string resultsDirectory) + { + Assert.IsTrue(File.Exists(trxFilePath), "Trx file not found: {0}", trxFilePath); + XmlDocument doc = new XmlDocument(); + using (var trxStream = new FileStream(trxFilePath, FileMode.Open, FileAccess.Read)) + { + doc.Load(trxStream); + var deploymentElements = doc.GetElementsByTagName("Deployment"); + Assert.IsTrue(deploymentElements.Count == 1, + "None or more than one Deployment tags found in trx file:{0}", trxFilePath); + var deploymentDir = deploymentElements[0].Attributes.GetNamedItem("runDeploymentRoot")?.Value; + Assert.IsTrue(string.IsNullOrEmpty(deploymentDir) == false, + "runDeploymentRoot attribute not found in trx file:{0}", trxFilePath); + var collectors = doc.GetElementsByTagName("Collector"); + + string fileName = string.Empty; + for (int i = 0; i < collectors.Count; i++) + { + if (string.Equals(collectors[i].Attributes.GetNamedItem("collectorDisplayName").Value, + "Code Coverage", StringComparison.OrdinalIgnoreCase)) + { + fileName = collectors[i].FirstChild?.FirstChild?.FirstChild?.Attributes.GetNamedItem("href") + ?.Value; + } + } + + Assert.IsTrue(string.IsNullOrEmpty(fileName) == false, "Coverage file name not found in trx file: {0}", + trxFilePath); + return Path.Combine(resultsDirectory, deploymentDir, "In", fileName); + } + } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 8dba9d84f6..2036355e18 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -283,37 +283,5 @@ private void AssertSourceFileName(XmlNode module) Assert.IsTrue(found); } - - private static string GetCoverageFileNameFromTrx(string trxFilePath, string resultsDirectory) - { - Assert.IsTrue(File.Exists(trxFilePath), "Trx file not found: {0}", trxFilePath); - XmlDocument doc = new XmlDocument(); - using (var trxStream = new FileStream(trxFilePath, FileMode.Open, FileAccess.Read)) - { - doc.Load(trxStream); - var deploymentElements = doc.GetElementsByTagName("Deployment"); - Assert.IsTrue(deploymentElements.Count == 1, - "None or more than one Deployment tags found in trx file:{0}", trxFilePath); - var deploymentDir = deploymentElements[0].Attributes.GetNamedItem("runDeploymentRoot")?.Value; - Assert.IsTrue(string.IsNullOrEmpty(deploymentDir) == false, - "runDeploymentRoot attribute not found in trx file:{0}", trxFilePath); - var collectors = doc.GetElementsByTagName("Collector"); - - string fileName = string.Empty; - for (int i = 0; i < collectors.Count; i++) - { - if (string.Equals(collectors[i].Attributes.GetNamedItem("collectorDisplayName").Value, - "Code Coverage", StringComparison.OrdinalIgnoreCase)) - { - fileName = collectors[i].FirstChild?.FirstChild?.FirstChild?.Attributes.GetNamedItem("href") - ?.Value; - } - } - - Assert.IsTrue(string.IsNullOrEmpty(fileName) == false, "Coverage file name not found in trx file: {0}", - trxFilePath); - return Path.Combine(resultsDirectory, deploymentDir, "In", fileName); - } - } } } \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs new file mode 100644 index 0000000000..304b285919 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests +{ + using System; + using System.IO; + using System.IO.Compression; + using System.Linq; + + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TestPlatformNugetPackageTests : CodeCoverageAcceptanceTestBase + { + private static string nugetPackageFolder; + private string resultsDirectory; + + [ClassInitialize] + public static void ClassInit(TestContext testContext) + { + var packageLocation = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "packages"); + var nugetPackage = Directory.EnumerateFiles(packageLocation, "Microsoft.TestPlatform.*.nupkg").OrderBy(a => a).FirstOrDefault(); + nugetPackageFolder = Path.Combine(packageLocation, Path.GetFileNameWithoutExtension(nugetPackage)); + ZipFile.ExtractToDirectory(nugetPackage, nugetPackageFolder); + + Directory.Move(Path.Combine(nugetPackageFolder, "tools", "net451", "Team%20Tools"), Path.Combine(nugetPackageFolder, "tools", "net451", "Team Tools")); + Directory.Move(Path.Combine(nugetPackageFolder, "tools", "net451", "Team Tools", "Dynamic%20Code%20Coverage%20Tools"), Path.Combine(nugetPackageFolder, "tools", "net451", "Team Tools", "Dynamic Code Coverage Tools")); + } + + [ClassCleanup] + public static void ClassCleanup() + { + //Directory.Delete(nugetPackageFolder, true); + } + + [TestInitialize] + public void SetUp() + { + this.resultsDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + } + + [TestCleanup] + public void CleanUp() + { + //Directory.Delete(resultsDirectory, true); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(useCoreRunner: false)] + [NetCoreTargetFrameworkDataSource(useCoreRunner: false)] + public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); + + var assemblyPaths = this.BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + + var arguments = CreateCodeCoverageArguments(runnerInfo, assemblyPaths, out var trxFilePath); + this.InvokeVsTest(arguments); + + this.ExitCodeEquals(1); // failing tests + + var actualCoverageFile = CodeCoverageTests.GetCoverageFileNameFromTrx(trxFilePath, resultsDirectory); + Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {resultsDirectory} trxfile: {trxFilePath}"); + Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); + + Directory.Delete(this.resultsDirectory, true); + } + + public override string GetConsoleRunnerPath() + { + string consoleRunnerPath = string.Empty; + + if (this.IsDesktopRunner()) + { + consoleRunnerPath = Path.Combine(nugetPackageFolder, "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe"); + } + + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + return consoleRunnerPath; + } + + private string CreateCodeCoverageArguments( + RunnerInfo runnerInfo, + string assemblyPaths, + out string trxFilePath) + { + string diagFileName = Path.Combine(this.resultsDirectory, "diaglog.txt"); + + var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, + this.FrameworkArgValue, runnerInfo.InIsolationValue); + + arguments = string.Concat(arguments, $" /ResultsDirectory:{resultsDirectory}", $" /Diag:{diagFileName}", $" /EnableCodeCoverage"); + + trxFilePath = Path.Combine(this.resultsDirectory, Guid.NewGuid() + ".trx"); + arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); + + return arguments; + } + } +} From 3fa58277c29ef88446970f16ca8e4fdb94b37214 Mon Sep 17 00:00:00 2001 From: Jakub Chocholowicz Date: Mon, 23 Nov 2020 13:00:15 +0100 Subject: [PATCH 2/5] uncomment --- .../TestPlatformNugetPackageTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index 304b285919..dfce848c1d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -32,7 +32,7 @@ public static void ClassInit(TestContext testContext) [ClassCleanup] public static void ClassCleanup() { - //Directory.Delete(nugetPackageFolder, true); + Directory.Delete(nugetPackageFolder, true); } [TestInitialize] @@ -44,7 +44,7 @@ public void SetUp() [TestCleanup] public void CleanUp() { - //Directory.Delete(resultsDirectory, true); + Directory.Delete(resultsDirectory, true); } [TestMethod] From b798e8aedb9523c1285d7cf410f96da5cc2debae Mon Sep 17 00:00:00 2001 From: Jakub Chocholowicz Date: Mon, 23 Nov 2020 15:06:19 +0100 Subject: [PATCH 3/5] Add back interop for signing --- src/package/sign/sign.proj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/package/sign/sign.proj b/src/package/sign/sign.proj index 4173e79f43..f7082ad6fd 100644 --- a/src/package/sign/sign.proj +++ b/src/package/sign/sign.proj @@ -130,12 +130,13 @@ - + + From c2d28e05f27c965969d96431be4fab515588bd9c Mon Sep 17 00:00:00 2001 From: Jakub Chocholowicz Date: Mon, 23 Nov 2020 15:07:54 +0100 Subject: [PATCH 4/5] Downgrade even more --- scripts/build/TestPlatform.Dependencies.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 8c5a9d2835..7844e64ac3 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -31,7 +31,7 @@ 5.0.0 9.0.1 4.7.63 - 16.9.0-preview-4224421 + 16.9.0-preview-4189539 16.9.0-beta.20570.2 16.8.0-beta.20420.2 From 0e4e411cbdcc966dfcc3e5b4c8aaab832db7461f Mon Sep 17 00:00:00 2001 From: Jakub Chocholowicz Date: Mon, 23 Nov 2020 16:24:51 +0100 Subject: [PATCH 5/5] Fixing tests --- .../TestPlatformNugetPackageTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index dfce848c1d..80e920164c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -64,8 +64,6 @@ public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) var actualCoverageFile = CodeCoverageTests.GetCoverageFileNameFromTrx(trxFilePath, resultsDirectory); Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {resultsDirectory} trxfile: {trxFilePath}"); Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); - - Directory.Delete(this.resultsDirectory, true); } public override string GetConsoleRunnerPath()