From 0cdf8549f5d6a92c2ce8264c09481d384560fd9b Mon Sep 17 00:00:00 2001 From: Brad Wilson Date: Thu, 18 Apr 2024 13:59:27 -0700 Subject: [PATCH] Introduce a conservative parallelization algorithm (semaphore instead of sync context) --- src/common/ParallelAlgorithm.cs | 29 +++++ src/common/TestOptionsNames.cs | 1 + src/xunit.console/CommandLine.cs | 12 ++ src/xunit.console/ConsoleRunner.cs | 120 ++++++++++-------- .../TestFrameworkOptionsReadExtensions.cs | 18 +++ .../Frameworks/Runners/TestAssemblyRunner.cs | 5 +- .../Runners/XunitTestAssemblyRunner.cs | 23 +++- src/xunit.execution/xunit.execution.csproj | 1 + .../ConfigReader_Configuration.cs | 2 + .../Configuration/ConfigReader_Json.cs | 14 ++ ...TestFrameworkOptionsReadWriteExtensions.cs | 26 ++++ .../Frameworks/TestAssemblyConfiguration.cs | 10 ++ .../Frameworks/TestFrameworkOptions.cs | 1 + .../DefaultRunnerReporterMessageHandler.cs | 6 +- ...ltRunnerReporterWithTypesMessageHandler.cs | 6 +- .../Runners/AssemblyRunner.cs | 30 ++++- .../xunit.runner.utility.csproj | 1 + test/test.utility/TestDoubles/Mocks.cs | 4 +- .../Runners/XunitTestAssemblyRunnerTests.cs | 4 +- .../Common/ConfigReaderTests.cs | 6 + .../ConfigReader_BadValues.config | 3 +- .../ConfigReader_BadValues.json | 1 + .../ConfigReader_OverrideValues.config | 1 + .../ConfigReader_OverrideValues.json | 1 + 24 files changed, 254 insertions(+), 71 deletions(-) create mode 100644 src/common/ParallelAlgorithm.cs diff --git a/src/common/ParallelAlgorithm.cs b/src/common/ParallelAlgorithm.cs new file mode 100644 index 000000000..841458e0f --- /dev/null +++ b/src/common/ParallelAlgorithm.cs @@ -0,0 +1,29 @@ +#if XUNIT_FRAMEWORK +namespace Xunit.Sdk +#else +namespace Xunit +#endif +{ + /// + /// Indicates the parallelization algorithm to use. + /// + public enum ParallelAlgorithm + { + /// + /// The conservative parallelization algorithm uses a semaphore to limit the number of started tests to be equal + /// to the desired parallel thread count. This has the effect of allowing tests that have started to finish faster, + /// since there are no extra tests competing for a chance to run, at the expense that CPU utilization will be lowered + /// if the test project spaws a lot of async tests that have significant wait times. + /// + Conservative = 0, + + /// + /// The aggressive parallelization algorithm uses a synchronization context to limit the number of running tests + /// to be equal to the desired parallel thread count. This has the effect of being able to use the CPU more + /// effectively since there are typically most tests capable of running than there are CPU cores, at the + /// expense of tests that have already started being put into the back of a long queue before they can run + /// again. + /// + Aggressive = 1, + } +} diff --git a/src/common/TestOptionsNames.cs b/src/common/TestOptionsNames.cs index 38169d7e4..61e9c0484 100644 --- a/src/common/TestOptionsNames.cs +++ b/src/common/TestOptionsNames.cs @@ -17,6 +17,7 @@ internal static class Execution public static readonly string InternalDiagnosticMessages = "xunit.execution.InternalDiagnosticMessages"; public static readonly string DisableParallelization = "xunit.execution.DisableParallelization"; public static readonly string MaxParallelThreads = "xunit.execution.MaxParallelThreads"; + public static readonly string ParallelAlgorithm = "xunit.execution.ParallelAlgorithm"; public static readonly string SynchronousMessageReporting = "xunit.execution.SynchronousMessageReporting"; } } diff --git a/src/xunit.console/CommandLine.cs b/src/xunit.console/CommandLine.cs index 9eb9c853d..d7c89a284 100644 --- a/src/xunit.console/CommandLine.cs +++ b/src/xunit.console/CommandLine.cs @@ -46,6 +46,8 @@ protected CommandLine(string[] args, Predicate fileExists = null) public XunitProject Project { get; protected set; } + public ParallelAlgorithm? ParallelAlgorithm { get; protected set; } + public bool? ParallelizeAssemblies { get; protected set; } public bool? ParallelizeTestCollections { get; set; } @@ -298,6 +300,16 @@ protected XunitProject Parse(Predicate fileExists) break; } } + else if (optionName == "parallelalgorithm") + { + if (option.Value == null) + throw new ArgumentException("missing argument for -parallelAlgorithm"); + + if (!Enum.TryParse(option.Value, ignoreCase: true, out ParallelAlgorithm parallelAlgorithm)) + throw new ArgumentException("incorrect argument value for -parallelAlgorithm"); + + ParallelAlgorithm = parallelAlgorithm; + } else if (optionName == "noshadow") { GuardNoOptionValue(option); diff --git a/src/xunit.console/ConsoleRunner.cs b/src/xunit.console/ConsoleRunner.cs index e652e7d41..9a88d33ae 100644 --- a/src/xunit.console/ConsoleRunner.cs +++ b/src/xunit.console/ConsoleRunner.cs @@ -85,7 +85,8 @@ public int EntryPoint(string[] args) var failCount = RunProject(commandLine.Project, commandLine.Serialize, commandLine.ParallelizeAssemblies, commandLine.ParallelizeTestCollections, commandLine.MaxParallelThreads, commandLine.DiagnosticMessages, commandLine.NoColor, commandLine.AppDomains, - commandLine.FailSkips, commandLine.StopOnFail, commandLine.InternalDiagnosticMessages); + commandLine.FailSkips, commandLine.StopOnFail, commandLine.InternalDiagnosticMessages, + commandLine.ParallelAlgorithm); if (cancel) return -1073741510; // 0xC000013A: The application terminated as a result of a CTRL+C @@ -208,60 +209,63 @@ void PrintUsage(IReadOnlyList reporters) #endif Console.WriteLine(); Console.WriteLine("Valid options:"); - Console.WriteLine(" -nologo : do not show the copyright message"); - Console.WriteLine(" -nocolor : do not output results with colors"); - Console.WriteLine(" -failskips : convert skipped tests into failures"); - Console.WriteLine(" -stoponfail : stop on first test failure"); - Console.WriteLine(" -parallel option : set parallelization based on option"); - Console.WriteLine(" : none - turn off all parallelization"); - Console.WriteLine(" : collections - only parallelize collections"); - Console.WriteLine(" : assemblies - only parallelize assemblies"); - Console.WriteLine(" : all - parallelize assemblies & collections"); - Console.WriteLine(" -maxthreads count : maximum thread count for collection parallelization"); - Console.WriteLine(" : default - run with default (1 thread per CPU thread)"); - Console.WriteLine(" : unlimited - run with unbounded thread count"); - Console.WriteLine(" : (number) - limit task thread pool size to 'count'"); + Console.WriteLine(" -nologo : do not show the copyright message"); + Console.WriteLine(" -nocolor : do not output results with colors"); + Console.WriteLine(" -failskips : convert skipped tests into failures"); + Console.WriteLine(" -stoponfail : stop on first test failure"); + Console.WriteLine(" -parallel option : set parallelization based on option"); + Console.WriteLine(" : none - turn off all parallelization"); + Console.WriteLine(" : collections - only parallelize collections"); + Console.WriteLine(" : assemblies - only parallelize assemblies"); + Console.WriteLine(" : all - parallelize assemblies & collections"); + Console.WriteLine(" -parallelAlgorithm option : set the parallelization algoritm"); + Console.WriteLine(" : conservative - start the minimum number of tests (default)"); + Console.WriteLine(" : aggressive - start as many tests as possible"); + Console.WriteLine(" -maxthreads count : maximum thread count for collection parallelization"); + Console.WriteLine(" : default - run with default (1 thread per CPU thread)"); + Console.WriteLine(" : unlimited - run with unbounded thread count"); + Console.WriteLine(" : (number) - limit task thread pool size to 'count'"); #if NETFRAMEWORK - Console.WriteLine(" -appdomains mode : choose an app domain mode"); - Console.WriteLine(" : ifavailable - choose based on library type"); - Console.WriteLine(" : required - force app domains on"); - Console.WriteLine(" : denied - force app domains off"); - Console.WriteLine(" -noshadow : do not shadow copy assemblies"); + Console.WriteLine(" -appdomains mode : choose an app domain mode"); + Console.WriteLine(" : ifavailable - choose based on library type"); + Console.WriteLine(" : required - force app domains on"); + Console.WriteLine(" : denied - force app domains off"); + Console.WriteLine(" -noshadow : do not shadow copy assemblies"); #endif - Console.WriteLine(" -wait : wait for input after completion"); - Console.WriteLine(" -diagnostics : enable diagnostics messages for all test assemblies"); - Console.WriteLine(" -internaldiagnostics : enable internal diagnostics messages for all test assemblies"); + Console.WriteLine(" -wait : wait for input after completion"); + Console.WriteLine(" -diagnostics : enable diagnostics messages for all test assemblies"); + Console.WriteLine(" -internaldiagnostics : enable internal diagnostics messages for all test assemblies"); #if DEBUG - Console.WriteLine(" -pause : pause before doing any work, to help attach a debugger"); + Console.WriteLine(" -pause : pause before doing any work, to help attach a debugger"); #endif - Console.WriteLine(" -debug : launch the debugger to debug the tests"); - Console.WriteLine(" -serialize : serialize all test cases (for diagnostic purposes only)"); - Console.WriteLine(" -trait \"name=value\" : only run tests with matching name/value traits"); - Console.WriteLine(" : if specified more than once, acts as an OR operation"); - Console.WriteLine(" -notrait \"name=value\" : do not run tests with matching name/value traits"); - Console.WriteLine(" : if specified more than once, acts as an AND operation"); - Console.WriteLine(" -method \"name\" : run a given test method (can be fully specified or use a wildcard;"); - Console.WriteLine(" : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')"); - Console.WriteLine(" : if specified more than once, acts as an OR operation"); - Console.WriteLine(" -nomethod \"name\" : do not run a given test method (can be fully specified or use a wildcard;"); - Console.WriteLine(" : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')"); - Console.WriteLine(" : if specified more than once, acts as an AND operation"); - Console.WriteLine(" -class \"name\" : run all methods in a given test class (should be fully"); - Console.WriteLine(" : specified; i.e., 'MyNamespace.MyClass')"); - Console.WriteLine(" : if specified more than once, acts as an OR operation"); - Console.WriteLine(" -noclass \"name\" : do not run any methods in a given test class (should be fully"); - Console.WriteLine(" : specified; i.e., 'MyNamespace.MyClass')"); - Console.WriteLine(" : if specified more than once, acts as an AND operation"); - Console.WriteLine(" -namespace \"name\" : run all methods in a given namespace (i.e.,"); - Console.WriteLine(" : 'MyNamespace.MySubNamespace')"); - Console.WriteLine(" : if specified more than once, acts as an OR operation"); - Console.WriteLine(" -nonamespace \"name\" : do not run any methods in a given namespace (i.e.,"); - Console.WriteLine(" : 'MyNamespace.MySubNamespace')"); - Console.WriteLine(" : if specified more than once, acts as an AND operation"); - Console.WriteLine(" -noautoreporters : do not allow reporters to be auto-enabled by environment"); - Console.WriteLine(" : (for example, auto-detecting TeamCity or AppVeyor)"); + Console.WriteLine(" -debug : launch the debugger to debug the tests"); + Console.WriteLine(" -serialize : serialize all test cases (for diagnostic purposes only)"); + Console.WriteLine(" -trait \"name=value\" : only run tests with matching name/value traits"); + Console.WriteLine(" : if specified more than once, acts as an OR operation"); + Console.WriteLine(" -notrait \"name=value\" : do not run tests with matching name/value traits"); + Console.WriteLine(" : if specified more than once, acts as an AND operation"); + Console.WriteLine(" -method \"name\" : run a given test method (can be fully specified or use a wildcard;"); + Console.WriteLine(" : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')"); + Console.WriteLine(" : if specified more than once, acts as an OR operation"); + Console.WriteLine(" -nomethod \"name\" : do not run a given test method (can be fully specified or use a wildcard;"); + Console.WriteLine(" : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')"); + Console.WriteLine(" : if specified more than once, acts as an AND operation"); + Console.WriteLine(" -class \"name\" : run all methods in a given test class (should be fully"); + Console.WriteLine(" : specified; i.e., 'MyNamespace.MyClass')"); + Console.WriteLine(" : if specified more than once, acts as an OR operation"); + Console.WriteLine(" -noclass \"name\" : do not run any methods in a given test class (should be fully"); + Console.WriteLine(" : specified; i.e., 'MyNamespace.MyClass')"); + Console.WriteLine(" : if specified more than once, acts as an AND operation"); + Console.WriteLine(" -namespace \"name\" : run all methods in a given namespace (i.e.,"); + Console.WriteLine(" : 'MyNamespace.MySubNamespace')"); + Console.WriteLine(" : if specified more than once, acts as an OR operation"); + Console.WriteLine(" -nonamespace \"name\" : do not run any methods in a given namespace (i.e.,"); + Console.WriteLine(" : 'MyNamespace.MySubNamespace')"); + Console.WriteLine(" : if specified more than once, acts as an AND operation"); + Console.WriteLine(" -noautoreporters : do not allow reporters to be auto-enabled by environment"); + Console.WriteLine(" : (for example, auto-detecting TeamCity or AppVeyor)"); #if NETCOREAPP - Console.WriteLine(" -framework \"name\" : set the target framework"); + Console.WriteLine(" -framework \"name\" : set the target framework"); #endif Console.WriteLine(); @@ -271,14 +275,14 @@ void PrintUsage(IReadOnlyList reporters) Console.WriteLine("Reporters: (optional, choose only one)"); foreach (var reporter in switchableReporters.OrderBy(r => r.RunnerSwitch)) - Console.WriteLine(" -{0} : {1}", reporter.RunnerSwitch.ToLowerInvariant().PadRight(21), reporter.Description); + Console.WriteLine(" -{0} : {1}", reporter.RunnerSwitch.ToLowerInvariant().PadRight(24), reporter.Description); Console.WriteLine(); } Console.WriteLine("Result formats: (optional, choose one or more)"); TransformFactory.AvailableTransforms.ForEach( - transform => Console.WriteLine(" -{0} : {1}", string.Format(CultureInfo.CurrentCulture, "{0} ", transform.CommandLine).PadRight(21).Substring(0, 21), transform.Description) + transform => Console.WriteLine(" -{0} : {1}", string.Format(CultureInfo.CurrentCulture, "{0} ", transform.CommandLine).PadRight(24).Substring(0, 24), transform.Description) ); } @@ -292,7 +296,8 @@ void PrintUsage(IReadOnlyList reporters) AppDomainSupport? appDomains, bool failSkips, bool stopOnFail, - bool internalDiagnosticMessages) + bool internalDiagnosticMessages, + ParallelAlgorithm? parallelAlgorithm) { XElement assembliesElement = null; var clockTime = Stopwatch.StartNew(); @@ -309,7 +314,7 @@ void PrintUsage(IReadOnlyList reporters) if (parallelizeAssemblies.GetValueOrDefault()) { - var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages))); + var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) assembliesElement.Add(assemblyElement); @@ -318,7 +323,7 @@ void PrintUsage(IReadOnlyList reporters) { foreach (var assembly in project.Assemblies) { - var assemblyElement = ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages); + var assemblyElement = ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm); if (assemblyElement != null) assembliesElement.Add(assemblyElement); } @@ -351,7 +356,8 @@ void PrintUsage(IReadOnlyList reporters) bool failSkips, bool stopOnFail, XunitFilters filters, - bool internalDiagnosticMessages) + bool internalDiagnosticMessages, + ParallelAlgorithm? parallelAlgorithm) { foreach (var warning in assembly.ConfigWarnings) logger.LogWarning(warning); @@ -385,6 +391,8 @@ void PrintUsage(IReadOnlyList reporters) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); if (stopOnFail) executionOptions.SetStopOnTestFail(stopOnFail); + if (parallelAlgorithm.HasValue) + executionOptions.SetParallelAlgorithm(parallelAlgorithm); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); diff --git a/src/xunit.execution/Extensions/TestFrameworkOptionsReadExtensions.cs b/src/xunit.execution/Extensions/TestFrameworkOptionsReadExtensions.cs index 229bf1f4f..0541be319 100644 --- a/src/xunit.execution/Extensions/TestFrameworkOptionsReadExtensions.cs +++ b/src/xunit.execution/Extensions/TestFrameworkOptionsReadExtensions.cs @@ -119,6 +119,24 @@ public static bool DiagnosticMessagesOrDefault(this ITestFrameworkExecutionOptio return executionOptions.DiagnosticMessages() ?? false; } + /// + /// Gets the parallel algorithm to be used. + /// + public static ParallelAlgorithm? ParallelAlgorithm(this ITestFrameworkExecutionOptions executionOptions) + { + var parallelAlgorithmString = executionOptions.GetValue(TestOptionsNames.Execution.ParallelAlgorithm); + return parallelAlgorithmString != null ? (ParallelAlgorithm?)Enum.Parse(typeof(ParallelAlgorithm), parallelAlgorithmString) : null; + } + + /// + /// Gets the parallel algorithm to be used. If the flag is not present, return the default + /// value (). + /// + public static ParallelAlgorithm ParallelAlgorithmOrDefault(this ITestFrameworkExecutionOptions executionOptions) + { + return executionOptions.ParallelAlgorithm() ?? Xunit.Sdk.ParallelAlgorithm.Conservative; + } + /// /// Gets a flag to disable parallelization. /// diff --git a/src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs b/src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs index b638d966e..2a113a836 100644 --- a/src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs +++ b/src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; -using System.IO; using System.Linq; using System.Reflection; using System.Runtime.Versioning; @@ -10,6 +9,10 @@ using System.Threading.Tasks; using Xunit.Abstractions; +#if NETFRAMEWORK +using System.IO; +#endif + namespace Xunit.Sdk { /// diff --git a/src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs b/src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs index 6afc84e5e..3ff8f1000 100644 --- a/src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs +++ b/src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs @@ -18,7 +18,9 @@ public class XunitTestAssemblyRunner : TestAssemblyRunner bool disableParallelization; bool initialized; int maxParallelThreads; + ParallelAlgorithm parallelAlgorithm; SynchronizationContext originalSyncContext; + SemaphoreSlim parallelSemaphore; MaxConcurrencySyncContext syncContext; /// @@ -102,6 +104,8 @@ protected void Initialize() if (maxParallelThreads == 0) maxParallelThreads = Environment.ProcessorCount; + parallelAlgorithm = ExecutionOptions.ParallelAlgorithmOrDefault(); + var testCaseOrdererAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (testCaseOrdererAttribute != null) { @@ -191,7 +195,10 @@ protected override async Task RunTestCollectionsAsync(IMessageBus me if (disableParallelization) return await base.RunTestCollectionsAsync(messageBus, cancellationTokenSource); - SetupSyncContext(maxParallelThreads); + if (parallelAlgorithm == ParallelAlgorithm.Aggressive) + SetupSyncContext(maxParallelThreads); + else + parallelSemaphore = new SemaphoreSlim(maxParallelThreads); Func>, Task> taskRunner; if (SynchronizationContext.Current != null) @@ -259,7 +266,19 @@ protected override async Task RunTestCollectionsAsync(IMessageBus me /// protected override Task RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) - => new XunitTestCollectionRunner(testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync(); + { + + parallelSemaphore?.Wait(); + + try + { + return new XunitTestCollectionRunner(testCollection, testCases, DiagnosticMessageSink, messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), cancellationTokenSource).RunAsync(); + } + finally + { + parallelSemaphore?.Release(); + } + } [SecuritySafeCritical] static void SetSynchronizationContext(SynchronizationContext context) diff --git a/src/xunit.execution/xunit.execution.csproj b/src/xunit.execution/xunit.execution.csproj index f594d05cd..fa9592551 100644 --- a/src/xunit.execution/xunit.execution.csproj +++ b/src/xunit.execution/xunit.execution.csproj @@ -20,6 +20,7 @@ + diff --git a/src/xunit.runner.utility/Configuration/ConfigReader_Configuration.cs b/src/xunit.runner.utility/Configuration/ConfigReader_Configuration.cs index 02ea9a62a..04a208ba2 100644 --- a/src/xunit.runner.utility/Configuration/ConfigReader_Configuration.cs +++ b/src/xunit.runner.utility/Configuration/ConfigReader_Configuration.cs @@ -62,6 +62,7 @@ public static TestAssemblyConfiguration Load(string assemblyFileName, string con result.MaxParallelThreads = GetInt(settings, Configuration.MaxParallelThreads) ?? result.MaxParallelThreads; result.MethodDisplay = GetEnum(settings, Configuration.MethodDisplay) ?? result.MethodDisplay; result.MethodDisplayOptions = GetEnum(settings, Configuration.MethodDisplayOptions) ?? result.MethodDisplayOptions; + result.ParallelAlgorithm = GetEnum(settings, Configuration.ParallelAlgorithm) ?? result.ParallelAlgorithm; result.ParallelizeAssembly = GetBoolean(settings, Configuration.ParallelizeAssembly) ?? result.ParallelizeAssembly; result.ParallelizeTestCollections = GetBoolean(settings, Configuration.ParallelizeTestCollections) ?? result.ParallelizeTestCollections; result.PreEnumerateTheories = GetBoolean(settings, Configuration.PreEnumerateTheories) ?? result.PreEnumerateTheories; @@ -134,6 +135,7 @@ static class Configuration public const string MaxParallelThreads = "xunit.maxParallelThreads"; public const string MethodDisplay = "xunit.methodDisplay"; public const string MethodDisplayOptions = "xunit.methodDisplayOptions"; + public const string ParallelAlgorithm = "xunit.parallelAlgorithm"; public const string ParallelizeAssembly = "xunit.parallelizeAssembly"; public const string ParallelizeTestCollections = "xunit.parallelizeTestCollections"; public const string PreEnumerateTheories = "xunit.preEnumerateTheories"; diff --git a/src/xunit.runner.utility/Configuration/ConfigReader_Json.cs b/src/xunit.runner.utility/Configuration/ConfigReader_Json.cs index c189373ed..2126bcef1 100644 --- a/src/xunit.runner.utility/Configuration/ConfigReader_Json.cs +++ b/src/xunit.runner.utility/Configuration/ConfigReader_Json.cs @@ -160,6 +160,19 @@ static TestAssemblyConfiguration LoadConfiguration(Stream configStream, string c catch { } } } + else if (string.Equals(propertyName, Configuration.ParallelAlgorithm, StringComparison.OrdinalIgnoreCase)) + { + var stringValue = propertyValue as JsonString; + if (stringValue != null) + { + try + { + var parallelAlgorithm = Enum.Parse(typeof(ParallelAlgorithm), stringValue, true); + result.ParallelAlgorithm = (ParallelAlgorithm)parallelAlgorithm; + } + catch { } + } + } else if (string.Equals(propertyName, Configuration.AppDomain, StringComparison.OrdinalIgnoreCase)) { var stringValue = propertyValue as JsonString; @@ -237,6 +250,7 @@ static class Configuration public const string MaxParallelThreads = "maxParallelThreads"; public const string MethodDisplay = "methodDisplay"; public const string MethodDisplayOptions = "methodDisplayOptions"; + public const string ParallelAlgorithm = "parallelAlgorithm"; public const string ParallelizeAssembly = "parallelizeAssembly"; public const string ParallelizeTestCollections = "parallelizeTestCollections"; public const string PreEnumerateTheories = "preEnumerateTheories"; diff --git a/src/xunit.runner.utility/Extensions/TestFrameworkOptionsReadWriteExtensions.cs b/src/xunit.runner.utility/Extensions/TestFrameworkOptionsReadWriteExtensions.cs index eec105ddf..47f41678f 100644 --- a/src/xunit.runner.utility/Extensions/TestFrameworkOptionsReadWriteExtensions.cs +++ b/src/xunit.runner.utility/Extensions/TestFrameworkOptionsReadWriteExtensions.cs @@ -242,6 +242,24 @@ public static int GetMaxParallelThreadsOrDefault(this ITestFrameworkExecutionOpt return result.GetValueOrDefault(); } + /// + /// Gets the parallel algorithm to be used. + /// + public static ParallelAlgorithm? GetParallelAlgorithm(this ITestFrameworkExecutionOptions executionOptions) + { + var parallelAlgorithmString = executionOptions.GetValue(TestOptionsNames.Execution.ParallelAlgorithm); + return parallelAlgorithmString != null ? (ParallelAlgorithm?)Enum.Parse(typeof(ParallelAlgorithm), parallelAlgorithmString) : null; + } + + /// + /// Gets the parallel algorithm to be used. If the flag is not present, return the default + /// value (). + /// + public static ParallelAlgorithm GetParallelAlgorithmOrDefault(this ITestFrameworkExecutionOptions executionOptions) + { + return executionOptions.GetParallelAlgorithm() ?? ParallelAlgorithm.Conservative; + } + /// /// Gets a flag that determines whether xUnit.net stop testing when a test fails. /// @@ -292,6 +310,14 @@ public static void SetInternalDiagnosticMessages(this ITestFrameworkExecutionOpt executionOptions.SetValue(TestOptionsNames.Execution.InternalDiagnosticMessages, value); } + /// + /// Sets the parallel algorith to be used. + /// + public static void SetParallelAlgorithm(this ITestFrameworkExecutionOptions executionOptions, ParallelAlgorithm? value) + { + executionOptions.SetValue(TestOptionsNames.Execution.ParallelAlgorithm, value.HasValue ? value.GetValueOrDefault().ToString() : null); + } + /// /// Sets a flag that determines whether xUnit.net stop testing when a test fails. /// diff --git a/src/xunit.runner.utility/Frameworks/TestAssemblyConfiguration.cs b/src/xunit.runner.utility/Frameworks/TestAssemblyConfiguration.cs index 141f0e10a..3e20fc0ba 100644 --- a/src/xunit.runner.utility/Frameworks/TestAssemblyConfiguration.cs +++ b/src/xunit.runner.utility/Frameworks/TestAssemblyConfiguration.cs @@ -110,6 +110,16 @@ public int MaxParallelThreadsOrDefault /// public TestMethodDisplayOptions MethodDisplayOptionsOrDefault { get { return MethodDisplayOptions ?? TestMethodDisplayOptions.None; } } + /// + /// Gets or sets the algorithm to be used for parallelization. + /// + public ParallelAlgorithm? ParallelAlgorithm { get; set; } + + /// + /// Gets or sets the algorithm to be used for parallelization. + /// + public ParallelAlgorithm ParallelAlgorithmOrDefault { get { return ParallelAlgorithm ?? Xunit.ParallelAlgorithm.Conservative; } } + /// /// Gets or sets a flag indicating that this assembly is safe to parallelize against /// other assemblies. diff --git a/src/xunit.runner.utility/Frameworks/TestFrameworkOptions.cs b/src/xunit.runner.utility/Frameworks/TestFrameworkOptions.cs index bbdc828c0..5b2710132 100644 --- a/src/xunit.runner.utility/Frameworks/TestFrameworkOptions.cs +++ b/src/xunit.runner.utility/Frameworks/TestFrameworkOptions.cs @@ -52,6 +52,7 @@ public static ITestFrameworkExecutionOptions ForExecution(TestAssemblyConfigurat { result.SetDiagnosticMessages(configuration.DiagnosticMessages); result.SetInternalDiagnosticMessages(configuration.InternalDiagnosticMessages); + result.SetParallelAlgorithm(configuration.ParallelAlgorithm); result.SetDisableParallelization(!configuration.ParallelizeTestCollections); result.SetMaxParallelThreads(configuration.MaxParallelThreads); result.SetStopOnTestFail(configuration.StopOnFail); diff --git a/src/xunit.runner.utility/Reporters/DefaultRunnerReporterMessageHandler.cs b/src/xunit.runner.utility/Reporters/DefaultRunnerReporterMessageHandler.cs index d7c13b5ff..076a68082 100644 --- a/src/xunit.runner.utility/Reporters/DefaultRunnerReporterMessageHandler.cs +++ b/src/xunit.runner.utility/Reporters/DefaultRunnerReporterMessageHandler.cs @@ -230,14 +230,16 @@ protected override bool Visit(ITestAssemblyExecutionStarting executionStarting) if (executionStarting.ExecutionOptions.GetDiagnosticMessagesOrDefault()) { var threadCount = executionStarting.ExecutionOptions.GetMaxParallelThreadsOrDefault(); + var parallelAlgorithm = executionStarting.ExecutionOptions.GetParallelAlgorithmOrDefault(); var parallelTestCollections = executionStarting.ExecutionOptions.GetDisableParallelizationOrDefault() ? "off" : string.Format( CultureInfo.CurrentCulture, - "on [{0} thread{1}]", + "on [{0} thread{1}{2}]", threadCount < 0 ? "unlimited" : threadCount.ToString(CultureInfo.CurrentCulture), - threadCount == 1 ? string.Empty : "s" + threadCount == 1 ? string.Empty : "s", + parallelAlgorithm == ParallelAlgorithm.Aggressive ? "/aggressive" : string.Empty ); Logger.LogImportantMessage( diff --git a/src/xunit.runner.utility/Reporters/DefaultRunnerReporterWithTypesMessageHandler.cs b/src/xunit.runner.utility/Reporters/DefaultRunnerReporterWithTypesMessageHandler.cs index d0eed8d03..6ecbd3b2e 100644 --- a/src/xunit.runner.utility/Reporters/DefaultRunnerReporterWithTypesMessageHandler.cs +++ b/src/xunit.runner.utility/Reporters/DefaultRunnerReporterWithTypesMessageHandler.cs @@ -252,14 +252,16 @@ protected virtual void HandleTestAssemblyExecutionStarting(MessageHandlerArgs + /// Obsolete method. Call the overload with parallelAlgorithm. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("Please use the overload with parallelAlgorithm")] + public void Start(string typeName = null, + bool? diagnosticMessages = null, + TestMethodDisplay? methodDisplay = null, + TestMethodDisplayOptions? methodDisplayOptions = null, + bool? preEnumerateTheories = null, + bool? parallel = null, + int? maxParallelThreads = null, + bool? internalDiagnosticMessages = null) + { + Start(typeName, diagnosticMessages, methodDisplay, methodDisplayOptions, preEnumerateTheories, parallel, maxParallelThreads, internalDiagnosticMessages, null); + } + /// /// Starts running tests from a single type (if provided) or the whole assembly (if not). This call returns - /// immediately, and status results are dispatched to the Info>s on this class. Callers can check + /// immediately, and status results are dispatched to the events on this class. Callers can check /// to find out the current status. /// /// The (optional) type name of the single test class to run @@ -219,6 +239,7 @@ ITestFrameworkExecutionOptions GetExecutionOptions(bool? diagnosticMessages, boo /// of threads. By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.) /// Set to true to enable internal diagnostic messages; set to false to disable them. /// By default, uses the value from the assembly configuration file. + /// The parallel algorithm to be used; defaults to . public void Start(string typeName = null, bool? diagnosticMessages = null, TestMethodDisplay? methodDisplay = null, @@ -226,7 +247,8 @@ ITestFrameworkExecutionOptions GetExecutionOptions(bool? diagnosticMessages, boo bool? preEnumerateTheories = null, bool? parallel = null, int? maxParallelThreads = null, - bool? internalDiagnosticMessages = null) + bool? internalDiagnosticMessages = null, + ParallelAlgorithm? parallelAlgorithm = null) { lock (statusLock) { @@ -257,7 +279,7 @@ ITestFrameworkExecutionOptions GetExecutionOptions(bool? diagnosticMessages, boo return; } - var executionOptions = GetExecutionOptions(diagnosticMessages, parallel, maxParallelThreads, internalDiagnosticMessages); + var executionOptions = GetExecutionOptions(diagnosticMessages, parallel, parallelAlgorithm, maxParallelThreads, internalDiagnosticMessages); controller.RunTests(testCasesToRun, this, executionOptions); executionCompleteEvent.WaitOne(); }); diff --git a/src/xunit.runner.utility/xunit.runner.utility.csproj b/src/xunit.runner.utility/xunit.runner.utility.csproj index 936bd910f..449f8ef96 100644 --- a/src/xunit.runner.utility/xunit.runner.utility.csproj +++ b/src/xunit.runner.utility/xunit.runner.utility.csproj @@ -24,6 +24,7 @@ + diff --git a/test/test.utility/TestDoubles/Mocks.cs b/test/test.utility/TestDoubles/Mocks.cs index 80e2f354f..ee0be7313 100644 --- a/test/test.utility/TestDoubles/Mocks.cs +++ b/test/test.utility/TestDoubles/Mocks.cs @@ -247,10 +247,10 @@ public static ITestAssemblyExecutionFinished TestAssemblyExecutionFinished(bool return result; } - public static ITestAssemblyExecutionStarting TestAssemblyExecutionStarting(bool diagnosticMessages = false, string assemblyFilename = null, bool? parallelizeTestCollections = null, int maxParallelThreads = 42, bool? stopOnFail = null) + public static ITestAssemblyExecutionStarting TestAssemblyExecutionStarting(bool diagnosticMessages = false, string assemblyFilename = null, bool? parallelizeTestCollections = null, int maxParallelThreads = 42, bool? stopOnFail = null, Xunit.ParallelAlgorithm? parallelAlgorithm = null) { var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFilename ?? "testAssembly.dll", ConfigFilename = "testAssembly.dll.config" }; - var config = new TestAssemblyConfiguration { DiagnosticMessages = diagnosticMessages, MethodDisplay = Xunit.TestMethodDisplay.ClassAndMethod, MaxParallelThreads = maxParallelThreads, ParallelizeTestCollections = parallelizeTestCollections, ShadowCopy = true, StopOnFail = stopOnFail }; + var config = new TestAssemblyConfiguration { DiagnosticMessages = diagnosticMessages, MethodDisplay = Xunit.TestMethodDisplay.ClassAndMethod, MaxParallelThreads = maxParallelThreads, ParallelAlgorithm = parallelAlgorithm, ParallelizeTestCollections = parallelizeTestCollections, ShadowCopy = true, StopOnFail = stopOnFail }; var result = Substitute.For>(); result.Assembly.Returns(assembly); result.ExecutionOptions.Returns(TestFrameworkOptions.ForExecution(config)); diff --git a/test/test.xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunnerTests.cs b/test/test.xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunnerTests.cs index ee2ffff1a..67c7db296 100644 --- a/test/test.xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunnerTests.cs +++ b/test/test.xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunnerTests.cs @@ -8,6 +8,7 @@ using Xunit; using Xunit.Abstractions; using Xunit.Sdk; +using ParallelAlgorithm = Xunit.ParallelAlgorithm; public class XunitTestAssemblyRunnerTests { @@ -166,12 +167,13 @@ public static void TestOptionsOverrideAttribute() public class RunAsync { [Fact] - public static async void Parallel_SingleThread() + public static async void Parallel_SingleThread_Aggressive() { var passing = Mocks.XunitTestCase("Passing"); var other = Mocks.XunitTestCase("Other"); var options = TestFrameworkOptions.ForExecution(); options.SetMaxParallelThreads(1); + options.SetParallelAlgorithm(ParallelAlgorithm.Aggressive); var runner = TestableXunitTestAssemblyRunner.Create(testCases: new[] { passing, other }, executionOptions: options); await runner.RunAsync(); diff --git a/test/test.xunit.runner.utility/Common/ConfigReaderTests.cs b/test/test.xunit.runner.utility/Common/ConfigReaderTests.cs index aa238e618..250462ca1 100644 --- a/test/test.xunit.runner.utility/Common/ConfigReaderTests.cs +++ b/test/test.xunit.runner.utility/Common/ConfigReaderTests.cs @@ -63,6 +63,7 @@ public static void EmptyConfigurationFile_ReturnsDefaultValues() Assert.Equal(Environment.ProcessorCount, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.ClassAndMethod, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.None, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Conservative, result.ParallelAlgorithmOrDefault); Assert.False(result.ParallelizeAssemblyOrDefault); Assert.True(result.ParallelizeTestCollectionsOrDefault); Assert.True(result.PreEnumerateTheoriesOrDefault); @@ -82,6 +83,7 @@ public static void ConfigurationFileWithValidValues_ReturnsConfiguredValues() Assert.Equal(2112, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.Method, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.All, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Aggressive, result.ParallelAlgorithmOrDefault); Assert.True(result.ParallelizeAssemblyOrDefault); Assert.False(result.ParallelizeTestCollectionsOrDefault); Assert.False(result.PreEnumerateTheoriesOrDefault); @@ -102,6 +104,7 @@ public static void ConfigurationFileWithInvalidValues_FallsBackToDefaultValues() Assert.Equal(Environment.ProcessorCount, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.ClassAndMethod, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.None, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Conservative, result.ParallelAlgorithmOrDefault); // This value was valid as a sentinel to make sure we were trying to read values from the JSON Assert.True(result.ParallelizeAssemblyOrDefault); Assert.True(result.ParallelizeTestCollectionsOrDefault); @@ -157,6 +160,7 @@ public static void EmptyConfigurationFile_ReturnsDefaultValues() Assert.Equal(Environment.ProcessorCount, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.ClassAndMethod, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.None, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Conservative, result.ParallelAlgorithmOrDefault); Assert.False(result.ParallelizeAssemblyOrDefault); Assert.True(result.ParallelizeTestCollectionsOrDefault); Assert.True(result.PreEnumerateTheoriesOrDefault); @@ -175,6 +179,7 @@ public static void ConfigurationFileWithValidValues_ReturnsConfiguredValues() Assert.Equal(2112, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.Method, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.All, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Aggressive, result.ParallelAlgorithmOrDefault); Assert.True(result.ParallelizeAssemblyOrDefault); Assert.False(result.ParallelizeTestCollectionsOrDefault); Assert.False(result.PreEnumerateTheoriesOrDefault); @@ -194,6 +199,7 @@ public static void ConfigurationFileWithInvalidValues_FallsBackToDefaultValues() Assert.Equal(Environment.ProcessorCount, result.MaxParallelThreadsOrDefault); Assert.Equal(TestMethodDisplay.ClassAndMethod, result.MethodDisplayOrDefault); Assert.Equal(TestMethodDisplayOptions.None, result.MethodDisplayOptionsOrDefault); + Assert.Equal(ParallelAlgorithm.Conservative, result.ParallelAlgorithmOrDefault); // This value was valid as a sentinel to make sure we were trying to read values from the file Assert.True(result.ParallelizeAssemblyOrDefault); Assert.True(result.ParallelizeTestCollectionsOrDefault); diff --git a/test/test.xunit.runner.utility/ConfigReader_BadValues.config b/test/test.xunit.runner.utility/ConfigReader_BadValues.config index 516c90daa..e32436812 100644 --- a/test/test.xunit.runner.utility/ConfigReader_BadValues.config +++ b/test/test.xunit.runner.utility/ConfigReader_BadValues.config @@ -6,10 +6,11 @@ + - \ No newline at end of file + diff --git a/test/test.xunit.runner.utility/ConfigReader_BadValues.json b/test/test.xunit.runner.utility/ConfigReader_BadValues.json index 254a21322..ddf4975a1 100644 --- a/test/test.xunit.runner.utility/ConfigReader_BadValues.json +++ b/test/test.xunit.runner.utility/ConfigReader_BadValues.json @@ -5,6 +5,7 @@ "maxParallelThreads": "abc", "methodDisplay": "fooBar", "methodDisplayOptions": "fooBar", + "parallelAlgorithm": "blarch", "parallelizeAssembly": true, "parallelizetestcollections": "biff", "preEnumerateTheories": "baz" diff --git a/test/test.xunit.runner.utility/ConfigReader_OverrideValues.config b/test/test.xunit.runner.utility/ConfigReader_OverrideValues.config index 7e4f791cb..731923f70 100644 --- a/test/test.xunit.runner.utility/ConfigReader_OverrideValues.config +++ b/test/test.xunit.runner.utility/ConfigReader_OverrideValues.config @@ -6,6 +6,7 @@ + diff --git a/test/test.xunit.runner.utility/ConfigReader_OverrideValues.json b/test/test.xunit.runner.utility/ConfigReader_OverrideValues.json index d8cbfb27a..25667d479 100644 --- a/test/test.xunit.runner.utility/ConfigReader_OverrideValues.json +++ b/test/test.xunit.runner.utility/ConfigReader_OverrideValues.json @@ -5,6 +5,7 @@ "maxParallelThreads": 2112, "methodDisplay": "method", "methodDisplayOptions": "all", + "parallelAlgorithm": "aggressive", "parallelizeAssembly": true, "parallelizetestcollections": false, "preEnumerateTheories": false,