Fixed dotnet resolution for in-process vstest.console scenarios (#4122)

microsoft · Nov 22, 2022 · f098763 · f098763
1 parent 1bd4467
commit f098763
Show file tree

Hide file tree

Showing 7 changed files with 239 additions and 83 deletions.
diff --git a/src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetHostHelper.cs b/src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetHostHelper.cs
@@ -118,8 +118,14 @@ private bool TryGetExecutablePath(string executableBaseName, out string executab
         return false;
     }
 
-    public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitecture, [NotNullWhen(true)] out string? muxerPath)
+    public bool TryGetDotnetPathByArchitecture(
+        PlatformArchitecture targetArchitecture,
+        DotnetMuxerResolutionStrategy dotnetMuxerResolutionStrategy,
+        [NotNullWhen(true)] out string? muxerPath)
     {
+        muxerPath = null;
+        EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Using dotnet muxer resolution strategy: {dotnetMuxerResolutionStrategy}");
+
         // If current process is the same as the target architecture we return the current process filename.
         if (_processHelper.GetCurrentProcessArchitecture() == targetArchitecture)
         {
@@ -140,29 +146,37 @@ public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitectu
         bool isWinOs = _environment.OperatingSystem == PlatformOperatingSystem.Windows;
         EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Searching for muxer named '{_muxerName}'");
 
-        // Try to search using env vars in the order
-        // DOTNET_ROOT_{arch}
-        // DOTNET_ROOT(x86) if X86 on Win (here we cannot check if current process is WOW64 because this is SDK process arch and not real host arch so it's irrelevant)
-        //                  "DOTNET_ROOT(x86) is used instead when running a 32-bit executable on a 64-bit OS."
-        // DOTNET_ROOT
-        string envKey = $"DOTNET_ROOT_{targetArchitecture.ToString().ToUpperInvariant()}";
-
-        // Try on arch specific env var
-        string? envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
-
-        // Try on non virtualized x86 var(should happen only on non-x86 architecture)
-        if ((envVar == null || !_fileHelper.DirectoryExists(envVar)) &&
-            targetArchitecture == PlatformArchitecture.X86 && _environment.OperatingSystem == PlatformOperatingSystem.Windows)
+        string? envKey = null;
+        string? envVar = null;
+        if (dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.DotnetRootArchitecture))
         {
-            envKey = $"DOTNET_ROOT(x86)";
+            // Try to search using env vars in the order
+            // DOTNET_ROOT_{arch}
+            // DOTNET_ROOT(x86) if X86 on Win (here we cannot check if current process is WOW64 because this is SDK process arch and not real host arch so it's irrelevant)
+            //                  "DOTNET_ROOT(x86) is used instead when running a 32-bit executable on a 64-bit OS."
+            // DOTNET_ROOT
+            envKey = $"DOTNET_ROOT_{targetArchitecture.ToString().ToUpperInvariant()}";
+
+            // Try on arch specific env var
             envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
         }
 
-        // Try on default DOTNET_ROOT
-        if (envVar == null || !_fileHelper.DirectoryExists(envVar))
+        if (dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.DotnetRootArchitectureLess))
         {
-            envKey = "DOTNET_ROOT";
-            envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            // Try on non virtualized x86 var(should happen only on non-x86 architecture)
+            if ((envVar == null || !_fileHelper.DirectoryExists(envVar)) &&
+                targetArchitecture == PlatformArchitecture.X86 && _environment.OperatingSystem == PlatformOperatingSystem.Windows)
+            {
+                envKey = $"DOTNET_ROOT(x86)";
+                envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            }
+
+            // Try on default DOTNET_ROOT
+            if (envVar == null || !_fileHelper.DirectoryExists(envVar))
+            {
+                envKey = "DOTNET_ROOT";
+                envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            }
         }
 
         if (envVar != null)
@@ -196,64 +210,74 @@ public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitectu
             }
         }
 
-        EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer was not found using DOTNET_ROOT* env variables.");
-
-        // Try to search for global registration
-        muxerPath = isWinOs ? GetMuxerFromGlobalRegistrationWin(targetArchitecture) : GetMuxerFromGlobalRegistrationOnUnix(targetArchitecture);
+        if (dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.DotnetRootArchitecture)
+            || dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.DotnetRootArchitectureLess))
+        {
+            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer was not found using DOTNET_ROOT* env variables.");
+        }
 
-        if (muxerPath != null)
+        if (dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.GlobalInstallationLocation))
         {
-            if (!_fileHelper.Exists(muxerPath))
-            {
-                // If muxer doesn't exists or it's wrong we stop the search
-                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer file not found for global registration '{muxerPath}'");
-                muxerPath = null;
-                return false;
-            }
+            // Try to search for global registration
+            muxerPath = isWinOs ? GetMuxerFromGlobalRegistrationWin(targetArchitecture) : GetMuxerFromGlobalRegistrationOnUnix(targetArchitecture);
 
-            if (!IsValidArchitectureMuxer(targetArchitecture, muxerPath))
+            if (muxerPath != null)
             {
-                // If muxer is wrong we stop the search
-                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer resolved using global registration is not compatible with the target architecture: '{muxerPath}'");
-                muxerPath = null;
-                return false;
-            }
-
-            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer compatible with '{targetArchitecture}' resolved from global registration: '{muxerPath}'");
-            return true;
-        }
+                if (!_fileHelper.Exists(muxerPath))
+                {
+                    // If muxer doesn't exists or it's wrong we stop the search
+                    EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer file not found for global registration '{muxerPath}'");
+                    muxerPath = null;
+                    return false;
+                }
 
-        EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer not found using global registrations");
+                if (!IsValidArchitectureMuxer(targetArchitecture, muxerPath))
+                {
+                    // If muxer is wrong we stop the search
+                    EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer resolved using global registration is not compatible with the target architecture: '{muxerPath}'");
+                    muxerPath = null;
+                    return false;
+                }
 
-        // Try searching in default installation location if it exists
-        if (isWinOs)
-        {
-            // If we're on x64/arm64 SDK and target is x86 we need to search on non virtualized windows folder
-            if ((_environment.Architecture == PlatformArchitecture.X64 || _environment.Architecture == PlatformArchitecture.ARM64) &&
-                 targetArchitecture == PlatformArchitecture.X86)
-            {
-                muxerPath = Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles(x86)")!, "dotnet", _muxerName);
-            }
-            else
-            {
-                // If we're on ARM and target is x64 we expect correct installation inside x64 folder
-                muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
-                    ? Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", "x64", _muxerName)
-                    : Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", _muxerName);
+                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer compatible with '{targetArchitecture}' resolved from global registration: '{muxerPath}'");
+                return true;
             }
+
+            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer not found using global registrations");
         }
-        else
+
+        if (dotnetMuxerResolutionStrategy.HasFlag(DotnetMuxerResolutionStrategy.DefaultInstallationLocation))
         {
-            if (_environment.OperatingSystem == PlatformOperatingSystem.OSX)
+            // Try searching in default installation location if it exists
+            if (isWinOs)
             {
-                // If we're on ARM and target is x64 we expect correct installation inside x64 folder
-                muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
-                    ? Path.Combine("/usr/local/share/dotnet/x64", _muxerName)
-                    : Path.Combine("/usr/local/share/dotnet", _muxerName);
+                // If we're on x64/arm64 SDK and target is x86 we need to search on non virtualized windows folder
+                if ((_environment.Architecture == PlatformArchitecture.X64 || _environment.Architecture == PlatformArchitecture.ARM64) &&
+                     targetArchitecture == PlatformArchitecture.X86)
+                {
+                    muxerPath = Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles(x86)")!, "dotnet", _muxerName);
+                }
+                else
+                {
+                    // If we're on ARM and target is x64 we expect correct installation inside x64 folder
+                    muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
+                        ? Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", "x64", _muxerName)
+                        : Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", _muxerName);
+                }
             }
             else
             {
-                muxerPath = Path.Combine("/usr/share/dotnet", _muxerName);
+                if (_environment.OperatingSystem == PlatformOperatingSystem.OSX)
+                {
+                    // If we're on ARM and target is x64 we expect correct installation inside x64 folder
+                    muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
+                        ? Path.Combine("/usr/local/share/dotnet/x64", _muxerName)
+                        : Path.Combine("/usr/local/share/dotnet", _muxerName);
+                }
+                else
+                {
+                    muxerPath = Path.Combine("/usr/share/dotnet", _muxerName);
+                }
             }
         }
 

diff --git a/src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetMuxerResolutionStrategy.cs b/src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetMuxerResolutionStrategy.cs
@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
+
+/// <summary>
+/// An enum representing the dotnet muxer resolution.
+/// </summary>
+[Flags]
+public enum DotnetMuxerResolutionStrategy
+{
+    /// <summary>
+    /// Indicates if the muxer resolution process should take dotnet root into account.
+    /// </summary>
+    DotnetRootArchitecture = 1,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should take arch independent dotnet root into account.
+    /// </summary>
+    DotnetRootArchitectureLess = 2,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should look in the global installation location.
+    /// </summary>
+    GlobalInstallationLocation = 4,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should look in the default installation location.
+    /// </summary>
+    DefaultInstallationLocation = 8,
+
+    /// <summary>
+    /// Default muxer resolution strategy.
+    /// </summary>
+    Default = DotnetRootArchitecture | DotnetRootArchitectureLess | GlobalInstallationLocation | DefaultInstallationLocation,
+}
diff --git a/src/Microsoft.TestPlatform.CoreUtilities/Helpers/Interfaces/IDotnetHostHelper.cs b/src/Microsoft.TestPlatform.CoreUtilities/Helpers/Interfaces/IDotnetHostHelper.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
 using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
 
 namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces;
@@ -27,7 +28,11 @@ public interface IDotnetHostHelper
     /// Try to locate muxer of specific architecture
     /// </summary>
     /// <param name="targetArchitecture">Specific architecture</param>
+    /// <param name="dotnetMuxerResolutionStrategy">The dotnet muxer resolution strategy.</param>
     /// <param name="muxerPath">Path to the muxer</param>
     /// <returns>True if native muxer is found</returns>
-    bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitecture, out string? muxerPath);
+    bool TryGetDotnetPathByArchitecture(
+        PlatformArchitecture targetArchitecture,
+        DotnetMuxerResolutionStrategy dotnetMuxerResolutionStrategy,
+        out string? muxerPath);
 }
diff --git a/src/Microsoft.TestPlatform.CoreUtilities/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CoreUtilities/PublicAPI/PublicAPI.Shipped.txt
@@ -63,7 +63,6 @@ Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces.ITestPlatfo
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.GetDotnetPath() -> string!
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.GetMonoPath() -> string!
-Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, out string? muxerPath) -> bool
 Microsoft.VisualStudio.TestPlatform.ObjectModel.EqtTrace
 Microsoft.VisualStudio.TestPlatform.ObjectModel.ValidateArg
 Microsoft.VisualStudio.TestPlatform.ObjectModel.ValidateArgProperty
@@ -169,3 +168,11 @@ static Microsoft.VisualStudio.TestPlatform.Utilities.TimeSpanParser.TryParse(str
 virtual Microsoft.VisualStudio.TestPlatform.Utilities.JobQueue<T>.WaitForQueueToGetEmpty() -> bool
 static Microsoft.VisualStudio.TestPlatform.Utilities.MulticastDelegateUtilities.SafeInvoke(this System.Delegate? delegates, object? sender, object! args, string! traceDisplayName) -> void
 Microsoft.VisualStudio.TestPlatform.Utilities.JobQueue<T>.IsPaused.get -> bool
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.Default = Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DotnetRootArchitecture | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DotnetRootArchitectureLess | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.GlobalInstallationLocation | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DefaultInstallationLocation -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DefaultInstallationLocation = 8 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DotnetRootArchitecture = 1 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.DotnetRootArchitectureLess = 2 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy.GlobalInstallationLocation = 4 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy
+Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy dotnetMuxerResolutionStrategy, out string? muxerPath) -> bool
+Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolutionStrategy dotnetMuxerResolutionStrategy, out string? muxerPath) -> bool
diff --git a/src/Microsoft.TestPlatform.CoreUtilities/PublicAPI/net/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.CoreUtilities/PublicAPI/net/PublicAPI.Shipped.txt
@@ -51,7 +51,6 @@ Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.Dot
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.DotnetHostHelper(Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces.IFileHelper! fileHelper, Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces.IEnvironment! environment) -> void
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.GetDotnetPath() -> string!
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.GetMonoPath() -> string!
-Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.DotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, out string? muxerPath) -> bool
 Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.FileHelper
 Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.FileHelper.CopyFile(string! sourcePath, string! destinationPath) -> void
 Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.FileHelper.CreateDirectory(string! path) -> System.IO.DirectoryInfo!

diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs
@@ -431,8 +431,14 @@ public Task<bool> LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, Ca
                     EqtTrace.Verbose($"DotnetTestHostmanager: Forcing the search to x64 architecure, IsDefaultTargetArchitecture '{_runsettingHelper.IsDefaultTargetArchitecture}' OS '{_platformEnvironment.OperatingSystem}' framework '{_targetFramework}'");
                 }
 
+                // Check if DOTNET_ROOT resolution should be bypassed.
+                var shouldIgnoreDotnetRoot = (_environmentVariableHelper.GetEnvironmentVariable("VSTEST_IGNORE_DOTNET_ROOT")?.Trim() ?? "0") != "0";
+                var muxerResolutionStrategy = shouldIgnoreDotnetRoot
+                        ? (DotnetMuxerResolutionStrategy.GlobalInstallationLocation | DotnetMuxerResolutionStrategy.DefaultInstallationLocation)
+                        : DotnetMuxerResolutionStrategy.Default;
+
                 PlatformArchitecture finalTargetArchitecture = forceToX64 ? PlatformArchitecture.X64 : targetArchitecture;
-                if (!_dotnetHostHelper.TryGetDotnetPathByArchitecture(finalTargetArchitecture, out string? muxerPath))
+                if (!_dotnetHostHelper.TryGetDotnetPathByArchitecture(finalTargetArchitecture, muxerResolutionStrategy, out string? muxerPath))
                 {
                     string message = string.Format(CultureInfo.CurrentCulture, Resources.NoDotnetMuxerFoundForArchitecture, $"dotnet{(_platformEnvironment.OperatingSystem == PlatformOperatingSystem.Windows ? ".exe" : string.Empty)}", finalTargetArchitecture.ToString());
                     EqtTrace.Error(message);