Backport 17.6.2 fixes (#4533)

Co-authored-by: Amaury Levé <amauryleve@microsoft.com>

eng/Versions.props

@@ -83,8 +83,10 @@
     <VSTestConsoleMostDownloadedVersion>[16.6.1]</VSTestConsoleMostDownloadedVersion>
     <VSTestConsolePreviousStableVersion>[16.11.0]</VSTestConsolePreviousStableVersion>
     <VSTestConsoleLegacyStableVersion>[15.9.2]</VSTestConsoleLegacyStableVersion>
-    <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
-    <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
+    <!-- This version also needs to be updated in src\Microsoft.TestPlatform.ObjectModel\Microsoft.TestPlatform.ObjectModel.nuspec -->
+    <!-- This version needs to be the same or lower than <NuGetFrameworksPackageVersion> https://github.com/dotnet/sdk/blob/release/main/eng/Versions.props
+    (or the respective branch. It also needs to be on nuget.org. -->
+    <NuGetFrameworksVersion>6.5.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
     <JsonNetVersion>13.0.1</JsonNetVersion>
   </PropertyGroup>

src/Microsoft.TestPlatform.ObjectModel/Microsoft.TestPlatform.ObjectModel.nuspec

@@ -6,17 +6,17 @@
     <dependencies>
       <group targetFramework="net462">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
 
       <group targetFramework="netcoreapp3.1">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
 
       <group targetFramework="netstandard2.0">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
     </dependencies>
 

src/vstest.console/CommandLine/CommandLineOptions.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 
 using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors;
+using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
@@ -287,7 +288,8 @@ public void AddSource(string source)
                 string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex);
         }
         // Add the matching files to source list
-        _sources = _sources.Union(matchingFiles).ToList();
+        var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles);
+        _sources = _sources.Union(filteredFiles).ToList();
     }
 
     /// <summary>

src/vstest.console/TestPlatformHelpers/TestRequestManager.cs

@@ -166,7 +166,7 @@ public void ResetOptions()
         EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started.");
 
         // TODO: Normalize rest of the data on the request as well
-        discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List<string>();
+        discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList());
         discoveryPayload.RunSettings ??= "<RunSettings></RunSettings>";
 
         var runsettings = discoveryPayload.RunSettings;
@@ -274,6 +274,11 @@ public void ResetOptions()
     {
         EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
         testRunRequestPayload.RunSettings ??= "<RunSettings></RunSettings>";
+        if (testRunRequestPayload.Sources != null)
+        {
+            testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources);
+        }
+
         var runsettings = testRunRequestPayload.RunSettings;
 
         if (testRunRequestPayload.TestPlatformOptions != null)
@@ -1464,3 +1469,73 @@ private static List<string> GetSources(TestRunRequestPayload testRunRequestPaylo
         return sources;
     }
 }
+
+internal static class KnownPlatformSourceFilter
+{
+    // Running tests on AzureDevops, many projects use the default filter
+    // which includes all *test*.dll, this includes many of the TestPlatform dlls,
+    // which we cannot run, and don't want to attempt to run.
+    // The default filter also filters out !*TestAdapter*.dll but it is easy to forget
+    // so we skip the most used adapters here as well.
+    private static readonly HashSet<string> KnownPlatformSources = new(new string[]
+    {
+        "Microsoft.TestPlatform.AdapterUtilities.dll",
+        "Microsoft.TestPlatform.AdapterUtilities.resources.dll",
+        "Microsoft.TestPlatform.CommunicationUtilities.dll",
+        "Microsoft.TestPlatform.CommunicationUtilities.resources.dll",
+        "Microsoft.TestPlatform.CoreUtilities.dll",
+        "Microsoft.TestPlatform.CoreUtilities.resources.dll",
+        "Microsoft.TestPlatform.CrossPlatEngine.dll",
+        "Microsoft.TestPlatform.CrossPlatEngine.resources.dll",
+        "Microsoft.TestPlatform.PlatformAbstractions.dll",
+        "Microsoft.TestPlatform.Utilities.dll",
+        "Microsoft.TestPlatform.Utilities.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.Common.dll",
+        "Microsoft.VisualStudio.TestPlatform.Common.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.ObjectModel.dll",
+        "Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll",
+        "testhost.dll",
+
+        // NUnit
+        "NUnit3.TestAdapter.dll",
+
+        // XUnit
+        "xunit.runner.visualstudio.testadapter.dll",
+        "xunit.runner.visualstudio.dotnetcore.testadapter.dll",
+
+        // MSTest
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.resources.dll",
+        // For MSTest up to v3
+        "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.resources.dll",
+    }, StringComparer.OrdinalIgnoreCase);
+
+
+    internal static List<string> FilterKnownPlatformSources(List<string>? sources)
+    {
+        if (sources == null)
+        {
+            return new List<string>();
+        }
+
+        var filteredSources = new List<string>();
+        foreach (string source in sources)
+        {
+            if (KnownPlatformSources.Contains(Path.GetFileName(source)))
+            {
+                EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'");
+            }
+            else
+            {
+                filteredSources.Add(source);
+            }
+        }
+
+        return filteredSources;
+    }
+}

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs

@@ -123,19 +123,18 @@ public void DiscoverTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(Runn
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var firstAssetDirectory = Path.GetDirectoryName(GetAssetFullPath("MSTestProject1.dll"))!;
+        var testDll = GetAssetFullPath("MSTestProject1.dll");
+        var nonTestDll = GetTestDllForFramework("NetStandard2Library.dll", "netstandard2.0");
 
-        // Include all dlls from the assembly dir, as the default AzDO filter *test*.dll does.
-        var dlls = Directory.EnumerateFiles(firstAssetDirectory, "*test*.dll").ToArray();
-        var quotedDlls = string.Join(" ", dlls.Select(a => a.AddDoubleQuote()));
+        var testAndNonTestDll = new[] { testDll, nonTestDll };
+        var quotedDlls = string.Join(" ", testAndNonTestDll.Select(a => a.AddDoubleQuote()));
 
         var arguments = PrepareArguments(quotedDlls, GetTestAdapterPath(), string.Empty, framework: string.Empty, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path);
         arguments = string.Concat(arguments, " /listtests");
         arguments = string.Concat(arguments, " /logger:\"console;prefix=true\"");
         InvokeVsTest(arguments);
 
-        var portableAssembly = dlls.Single(a => a.EndsWith("Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll"));
-        StringAssert.Contains(StdOut, $"Skipping source: {portableAssembly} (.NETPortable,Version=v4.5,Profile=Profile259,");
+        StringAssert.Contains(StdOut, $"Skipping source: {nonTestDll} (.NETStandard,Version=v2.0,");
 
         ExitCodeEquals(0);
     }

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs

@@ -10,6 +10,8 @@
 
 using TestPlatform.TestUtilities;
 using System.Linq;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using FluentAssertions;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;
 
@@ -405,19 +407,114 @@ public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(Runne
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var firstAssetDirectory = Path.GetDirectoryName(GetAssetFullPath("MSTestProject1.dll"))!;
+        var testDll = GetAssetFullPath("MSTestProject1.dll");
+        var nonTestDll = GetTestDllForFramework("NetStandard2Library.dll", "netstandard2.0");
 
-        // Include all dlls from the assembly dir, as the default AzDO filter *test*.dll does.
-        var dlls = Directory.EnumerateFiles(firstAssetDirectory, "*test*.dll").ToArray();
-        var quotedDlls = string.Join(" ", dlls.Select(a => a.AddDoubleQuote()));
+        var testAndNonTestDll = new[] { testDll, nonTestDll };
+        var quotedDlls = string.Join(" ", testAndNonTestDll.Select(a => a.AddDoubleQuote()));
 
         var arguments = PrepareArguments(quotedDlls, GetTestAdapterPath(), string.Empty, framework: string.Empty, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path);
         arguments = string.Concat(arguments, " /logger:\"console;prefix=true\"");
         InvokeVsTest(arguments);
 
-        var portableAssembly = dlls.Single(a => a.EndsWith("Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll"));
-        StringAssert.Contains(StdOut, $"Skipping source: {portableAssembly} (.NETPortable,Version=v4.5,Profile=Profile259,");
+        StringAssert.Contains(StdOut, $"Skipping source: {nonTestDll} (.NETStandard,Version=v2.0,");
 
         ExitCodeEquals(1);
     }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource]
+    public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+        var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList();
+        fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed.");
+        fails.Last().Should().Be("Test Run Failed.");
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll"));
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out");
+    }
 }

test/TestAssets/NetStandard2Library/Class1.cs

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace NetStandard2Library;
+public class Class1
+{
+
+}

test/TestAssets/NetStandard2Library/NetStandard2Library.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+  </PropertyGroup>
+
+</Project>

test/TestAssets/TestAssets.sln

@@ -130,7 +130,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SerializeTestRunTestProject
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MultiHostTestExecutionProject", "MultiHostTestExecutionProject\MultiHostTestExecutionProject.csproj", "{CE6673DA-B50F-46DF-99A3-8A7C54DE9B61}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NonDll.TestAdapter", "NonDll.TestAdapter\NonDll.TestAdapter.csproj", "{429552A4-4C18-4355-94C5-80DC88C48405}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NonDll.TestAdapter", "NonDll.TestAdapter\NonDll.TestAdapter.csproj", "{429552A4-4C18-4355-94C5-80DC88C48405}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetStandard2Library", "NetStandard2Library\NetStandard2Library.csproj", "{080F0AD2-E7AE-42C8-B867-56D78576735D}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -378,6 +380,10 @@ Global
 		{429552A4-4C18-4355-94C5-80DC88C48405}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{429552A4-4C18-4355-94C5-80DC88C48405}.Release|Any CPU.Build.0 = Release|Any CPU
+		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{080F0AD2-E7AE-42C8-B867-56D78576735D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE