Add ability to provide live output messages from running tests

src/common/TestOptionsNames.cs

@@ -18,6 +18,7 @@ internal static class Execution
         public static readonly string DisableParallelization = "xunit.execution.DisableParallelization";
         public static readonly string MaxParallelThreads = "xunit.execution.MaxParallelThreads";
         public static readonly string ParallelAlgorithm = "xunit.execution.ParallelAlgorithm";
+        public static readonly string ShowLiveOutput = "xunit.execution.ShowLiveOutput";
         public static readonly string SynchronousMessageReporting = "xunit.execution.SynchronousMessageReporting";
     }
 }

src/xunit.console/CommandLine.cs

@@ -54,6 +54,8 @@ protected CommandLine(string[] args, Predicate<string> fileExists = null)
 
         public bool Serialize { get; protected set; }
 
+        public bool ShowLiveOutput { get; protected set; }
+
         public bool StopOnFail { get; protected set; }
 
         public bool UseAnsiColor { get; protected set; }
@@ -166,6 +168,11 @@ protected XunitProject Parse(Predicate<string> fileExists)
                     GuardNoOptionValue(option);
                     FailSkips = true;
                 }
+                else if (optionName == "showliveoutput")
+                {
+                    GuardNoOptionValue(option);
+                    ShowLiveOutput = true;
+                }
                 else if (optionName == "stoponfail")
                 {
                     GuardNoOptionValue(option);

src/xunit.console/ConsoleRunner.cs

@@ -89,7 +89,7 @@ public int EntryPoint(string[] args)
                                            commandLine.ParallelizeTestCollections, commandLine.MaxParallelThreads,
                                            commandLine.DiagnosticMessages, commandLine.NoColor, commandLine.AppDomains,
                                            commandLine.FailSkips, commandLine.StopOnFail, commandLine.InternalDiagnosticMessages,
-                                           commandLine.ParallelAlgorithm);
+                                           commandLine.ParallelAlgorithm, commandLine.ShowLiveOutput);
 
                 if (cancel)
                     return -1073741510;    // 0xC000013A: The application terminated as a result of a CTRL+C
@@ -239,6 +239,7 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
             Console.WriteLine("  -wait                     : wait for input after completion");
             Console.WriteLine("  -diagnostics              : enable diagnostics messages for all test assemblies");
             Console.WriteLine("  -internaldiagnostics      : enable internal diagnostics messages for all test assemblies");
+            Console.WriteLine("  -showliveoutput           : show output messages from tests live");
 #if DEBUG
             Console.WriteLine("  -pause                    : pause before doing any work, to help attach a debugger");
 #endif
@@ -302,7 +303,8 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
                        bool failSkips,
                        bool stopOnFail,
                        bool internalDiagnosticMessages,
-                       ParallelAlgorithm? parallelAlgorithm)
+                       ParallelAlgorithm? parallelAlgorithm,
+                       bool showLiveOutput)
         {
             XElement assembliesElement = null;
             var clockTime = Stopwatch.StartNew();
@@ -319,7 +321,7 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
 
             if (parallelizeAssemblies.GetValueOrDefault())
             {
-                var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm)));
+                var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm, showLiveOutput)));
                 var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
                 foreach (var assemblyElement in results.Where(result => result != null))
                     assembliesElement.Add(assemblyElement);
@@ -328,7 +330,7 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
             {
                 foreach (var assembly in project.Assemblies)
                 {
-                    var assemblyElement = ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm);
+                    var assemblyElement = ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, appDomains, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages, parallelAlgorithm, showLiveOutput);
                     if (assemblyElement != null)
                         assembliesElement.Add(assemblyElement);
                 }
@@ -362,7 +364,8 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
                                  bool stopOnFail,
                                  XunitFilters filters,
                                  bool internalDiagnosticMessages,
-                                 ParallelAlgorithm? parallelAlgorithm)
+                                 ParallelAlgorithm? parallelAlgorithm,
+                                 bool showLiveOutput)
         {
             foreach (var warning in assembly.ConfigWarnings)
                 logger.LogWarning(warning);
@@ -394,6 +397,8 @@ void PrintUsage(IReadOnlyList<IRunnerReporter> reporters)
                     executionOptions.SetMaxParallelThreads(maxThreadCount);
                 if (parallelizeTestCollections.HasValue)
                     executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
+                if (showLiveOutput)
+                    executionOptions.SetShowLiveOutput(showLiveOutput);
                 if (stopOnFail)
                     executionOptions.SetStopOnTestFail(stopOnFail);
                 if (parallelAlgorithm.HasValue)

src/xunit.execution/Extensions/TestFrameworkOptionsReadExtensions.cs

@@ -176,6 +176,25 @@ public static int MaxParallelThreadsOrDefault(this ITestFrameworkExecutionOption
         return result.GetValueOrDefault();
     }
 
+    /// <summary>
+    /// Gets a flag which indicates if the developer wishes to see output from <see cref="ITestOutputHelper"/>
+    /// live while it's being reported (in addition to seeing it collected together when the test is finished).
+    /// </summary>
+    public static bool? ShowLiveOutput(this ITestFrameworkExecutionOptions executionOptions)
+    {
+        return executionOptions.GetValue<bool?>(TestOptionsNames.Execution.ShowLiveOutput);
+    }
+
+    /// <summary>
+    /// Gets a flag which indicates if the developer wishes to see output from <see cref="ITestOutputHelper"/>
+    /// live while it's being reported (in addition to seeing it collected together when the test is finished).
+    /// If the flag is not present, returns the default value (<c>false</c>).
+    /// </summary>
+    public static bool ShowLiveOutputOrDefault(this ITestFrameworkExecutionOptions executionOptions)
+    {
+        return executionOptions.ShowLiveOutput() ?? false;
+    }
+
     /// <summary>
     /// Gets a flag to stop testing on test failure.
     /// </summary>

src/xunit.runner.msbuild/xunit.cs

@@ -25,6 +25,7 @@ public class xunit : MSBuildTask, ICancelableTask
         bool? parallelizeTestCollections;
         IMessageSinkWithTypes reporterMessageHandler;
         bool? shadowCopy;
+        bool? showLiveOutput;
         bool? stopOnFail;
 
         public string AppDomains { get; set; }
@@ -89,6 +90,8 @@ protected bool NeedsXml
 
         public bool ShadowCopy { set { shadowCopy = value; } }
 
+        public bool ShowLiveOutput { set { showLiveOutput = value; } }
+
         public bool StopOnFail { set { stopOnFail = value; } }
 
         public string WorkingFolder { get; set; }
@@ -284,6 +287,8 @@ protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly, AppDom
                     executionOptions.SetParallelAlgorithm(parallelAlgorithm);
                 if (parallelizeTestCollections.HasValue)
                     executionOptions.SetDisableParallelization(!parallelizeTestCollections);
+                if (showLiveOutput.HasValue)
+                    executionOptions.SetShowLiveOutput(showLiveOutput);
                 if (stopOnFail.HasValue)
                     executionOptions.SetStopOnTestFail(stopOnFail);
 

src/xunit.runner.reporters/VerboseReporterMessageHandler.cs

@@ -5,7 +5,6 @@ public class VerboseReporterMessageHandler : DefaultRunnerReporterWithTypesMessa
         public VerboseReporterMessageHandler(IRunnerLogger logger)
             : base(logger)
         {
-            Execution.TestOutputEvent += args => Logger.LogMessage("    {0} [OUTPUT] {1}", Escape(args.Message.Test.DisplayName), Escape(args.Message.Output.TrimEnd('\r', '\n')));
             Execution.TestStartingEvent += args => Logger.LogMessage("    {0} [STARTING]", Escape(args.Message.Test.DisplayName));
             Execution.TestFinishedEvent += args => Logger.LogMessage("    {0} [FINISHED] Time: {1}s", Escape(args.Message.Test.DisplayName), args.Message.ExecutionTime);
         }

src/xunit.runner.utility/Configuration/ConfigReader_Configuration.cs

@@ -132,6 +132,7 @@ static class Configuration
             public const string AppDomain = "xunit.appDomain";
             public const string DiagnosticMessages = "xunit.diagnosticMessages";
             public const string InternalDiagnosticMessages = "xunit.internalDiagnosticMessages";
+            public const string LongRunningTestSeconds = "xunit.longRunningTestSeconds";
             public const string MaxParallelThreads = "xunit.maxParallelThreads";
             public const string MethodDisplay = "xunit.methodDisplay";
             public const string MethodDisplayOptions = "xunit.methodDisplayOptions";
@@ -141,7 +142,6 @@ static class Configuration
             public const string PreEnumerateTheories = "xunit.preEnumerateTheories";
             public const string ShadowCopy = "xunit.shadowCopy";
             public const string StopOnFail = "xunit.stopOnFail";
-            public const string LongRunningTestSeconds = "xunit.longRunningTestSeconds";
         }
     }
 }

src/xunit.runner.utility/Configuration/ConfigReader_Json.cs

@@ -111,6 +111,8 @@ static TestAssemblyConfiguration LoadConfiguration(Stream configStream, string c
                                 result.PreEnumerateTheories = booleanValue;
                             if (string.Equals(propertyName, Configuration.ShadowCopy, StringComparison.OrdinalIgnoreCase))
                                 result.ShadowCopy = booleanValue;
+                            if (string.Equals(propertyName, Configuration.ShowLiveOutput, StringComparison.OrdinalIgnoreCase))
+                                result.ShowLiveOutput = booleanValue;
                             if (string.Equals(propertyName, Configuration.StopOnFail, StringComparison.OrdinalIgnoreCase))
                                 result.StopOnFail = booleanValue;
                         }
@@ -270,6 +272,7 @@ static class Configuration
             public const string ParallelizeTestCollections = "parallelizeTestCollections";
             public const string PreEnumerateTheories = "preEnumerateTheories";
             public const string ShadowCopy = "shadowCopy";
+            public const string ShowLiveOutput = "showLiveOutput";
             public const string StopOnFail = "stopOnFail";
         }
     }

src/xunit.runner.utility/Extensions/TestFrameworkOptionsReadWriteExtensions.cs

@@ -260,6 +260,25 @@ public static ParallelAlgorithm GetParallelAlgorithmOrDefault(this ITestFramewor
         return executionOptions.GetParallelAlgorithm() ?? ParallelAlgorithm.Conservative;
     }
 
+    /// <summary>
+    /// Gets a flag which indicates if the developer wishes to see output from <see cref="ITestOutputHelper"/>
+    /// live while it's being reported (in addition to seeing it collected together when the test is finished).
+    /// </summary>
+    public static bool? GetShowLiveOutput(this ITestFrameworkExecutionOptions executionOptions)
+    {
+        return executionOptions.GetValue<bool?>(TestOptionsNames.Execution.ShowLiveOutput);
+    }
+
+    /// <summary>
+    /// Gets a flag which indicates if the developer wishes to see output from <see cref="ITestOutputHelper"/>
+    /// live while it's being reported (in addition to seeing it collected together when the test is finished).
+    /// If the flag is not present, returns the default value (<c>false</c>).
+    /// </summary>
+    public static bool GetShowLiveOutputOrDefault(this ITestFrameworkExecutionOptions executionOptions)
+    {
+        return executionOptions.GetShowLiveOutput() ?? false;
+    }
+
     /// <summary>
     /// Gets a flag that determines whether xUnit.net stop testing when a test fails.
     /// </summary>
@@ -318,6 +337,15 @@ public static void SetParallelAlgorithm(this ITestFrameworkExecutionOptions exec
         executionOptions.SetValue(TestOptionsNames.Execution.ParallelAlgorithm, value.HasValue ? value.GetValueOrDefault().ToString() : null);
     }
 
+    /// <summary>
+    /// Sets a flag which indicates if the developer wishes to see output from <see cref="ITestOutputHelper"/>
+    /// live while it's being reported (in addition to seeing it collected together when the test is finished).
+    /// </summary>
+    public static void SetShowLiveOutput(this ITestFrameworkExecutionOptions executionOptions, bool? value)
+    {
+        executionOptions.SetValue(TestOptionsNames.Execution.ShowLiveOutput, value);
+    }
+
     /// <summary>
     /// Sets a flag that determines whether xUnit.net stop testing when a test fails.
     /// </summary>

src/xunit.runner.utility/Frameworks/TestAssemblyConfiguration.cs

@@ -1,4 +1,5 @@
 using System;
+using Xunit.Abstractions;
 
 namespace Xunit
 {
@@ -168,6 +169,19 @@ public int MaxParallelThreadsOrDefault
         /// </summary>
         public bool ShadowCopyOrDefault { get { return ShadowCopy ?? true; } }
 
+        /// <summary>
+        /// Gets or sets a flag indicating whether output from <see cref="ITestOutputHelper"/> should be
+        /// shown live as they're logged (in addition to being collected together after the test finishes).
+        /// </summary>
+        public bool? ShowLiveOutput { get; set; }
+
+        /// <summary>
+        /// Gets a flag indicating whether output from <see cref="ITestOutputHelper"/> should be
+        /// shown live as they're logged (in addition to being collected together after the test finishes).
+        /// If the flag is not set, returns the default value (<c>false</c>).
+        /// </summary>
+        public bool ShowLiveOutputOrDefault { get { return ShowLiveOutput ?? false; } }
+
         /// <summary>
         /// Gets or sets a flag indicating whether testing should stop on a failure.
         /// </summary>

src/xunit.runner.utility/Frameworks/TestFrameworkOptions.cs

@@ -55,6 +55,7 @@ public static ITestFrameworkExecutionOptions ForExecution(TestAssemblyConfigurat
                 result.SetParallelAlgorithm(configuration.ParallelAlgorithm);
                 result.SetDisableParallelization(!configuration.ParallelizeTestCollections);
                 result.SetMaxParallelThreads(configuration.MaxParallelThreads);
+                result.SetShowLiveOutput(configuration.ShowLiveOutput);
                 result.SetStopOnTestFail(configuration.StopOnFail);
             }
 

src/xunit.runner.utility/Reporters/DefaultRunnerReporterMessageHandler.cs

@@ -330,6 +330,15 @@ protected override bool Visit(ITestMethodCleanupFailure cleanupFailure)
             return base.Visit(cleanupFailure);
         }
 
+        /// <inheritdoc/>
+        protected override bool Visit(ITestOutput testCaseOutput)
+        {
+            if (GetExecutionOptions(testCaseOutput.TestAssembly.Assembly.AssemblyPath).GetShowLiveOutputOrDefault())
+                Logger.LogMessage("    {0} [OUTPUT] {1}", Escape(testCaseOutput.Test.DisplayName), Escape(testCaseOutput.Output.TrimEnd()));
+
+            return base.Visit(testCaseOutput);
+        }
+
         /// <inheritdoc/>
         protected override bool Visit(ITestPassed testPassed)
         {

src/xunit.runner.utility/Reporters/DefaultRunnerReporterWithTypesMessageHandler.cs

@@ -39,6 +39,7 @@ public DefaultRunnerReporterWithTypesMessageHandler(IRunnerLogger logger)
             Execution.TestCleanupFailureEvent += HandleTestCleanupFailure;
             Execution.TestFailedEvent += HandleTestFailed;
             Execution.TestMethodCleanupFailureEvent += HandleTestMethodCleanupFailure;
+            Execution.TestOutputEvent += HandleTestOutput;
             Execution.TestPassedEvent += HandleTestPassed;
             Execution.TestSkippedEvent += HandleTestSkipped;
 
@@ -345,6 +346,17 @@ protected virtual void HandleTestFailed(MessageHandlerArgs<ITestFailed> args)
         protected virtual void HandleTestMethodCleanupFailure(MessageHandlerArgs<ITestMethodCleanupFailure> args)
             => LogError(string.Format(CultureInfo.CurrentCulture, "Test Method Cleanup Failure ({0})", args.Message.TestMethod.Method.Name), args.Message);
 
+        /// <summary>
+        /// Called when <see cref="ITestOutput"/> is raised.
+        /// </summary>
+        /// <param name="args">An object that contains the event data.</param>
+        protected virtual void HandleTestOutput(MessageHandlerArgs<ITestOutput> args)
+        {
+            var testOutput = args.Message;
+            if (GetExecutionOptions(testOutput.TestAssembly.Assembly.AssemblyPath).GetShowLiveOutputOrDefault())
+                Logger.LogMessage("    {0} [OUTPUT] {1}", Escape(testOutput.Test.DisplayName), Escape(testOutput.Output.TrimEnd()));
+        }
+
         /// <summary>
         /// Called when <see cref="ITestPassed"/> is raised.
         /// </summary>

test/test.utility/TestDoubles/Mocks.cs

@@ -259,10 +259,10 @@ public static ITestAssemblyExecutionFinished TestAssemblyExecutionFinished(bool
         return result;
     }
 
-    public static ITestAssemblyExecutionStarting TestAssemblyExecutionStarting(bool diagnosticMessages = false, string assemblyFilename = null, bool? parallelizeTestCollections = null, int maxParallelThreads = 42, bool? stopOnFail = null, Xunit.ParallelAlgorithm? parallelAlgorithm = null)
+    public static ITestAssemblyExecutionStarting TestAssemblyExecutionStarting(bool diagnosticMessages = false, string assemblyFilename = null, bool? parallelizeTestCollections = null, int maxParallelThreads = 42, bool? stopOnFail = null, Xunit.ParallelAlgorithm? parallelAlgorithm = null, bool? showLiveOutput = null)
     {
         var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFilename ?? "testAssembly.dll", ConfigFilename = "testAssembly.dll.config" };
-        var config = new TestAssemblyConfiguration { DiagnosticMessages = diagnosticMessages, MethodDisplay = Xunit.TestMethodDisplay.ClassAndMethod, MaxParallelThreads = maxParallelThreads, ParallelAlgorithm = parallelAlgorithm, ParallelizeTestCollections = parallelizeTestCollections, ShadowCopy = true, StopOnFail = stopOnFail };
+        var config = new TestAssemblyConfiguration { DiagnosticMessages = diagnosticMessages, MethodDisplay = Xunit.TestMethodDisplay.ClassAndMethod, MaxParallelThreads = maxParallelThreads, ParallelAlgorithm = parallelAlgorithm, ParallelizeTestCollections = parallelizeTestCollections, ShadowCopy = true, ShowLiveOutput = showLiveOutput, StopOnFail = stopOnFail };
         var result = Substitute.For<ITestAssemblyExecutionStarting, InterfaceProxy<ITestAssemblyExecutionStarting>>();
         result.Assembly.Returns(assembly);
         result.ExecutionOptions.Returns(TestFrameworkOptions.ForExecution(config));
@@ -501,6 +501,15 @@ public static TestMethod TestMethod(Type type, string methodName, ITestCollectio
         return new TestMethod(@class, Reflector.Wrap(methodInfo));
     }
 
+    public static ITestOutput TestOutput(string assemblyPath, string testDisplayName, string output)
+    {
+        var result = Substitute.For<ITestOutput, InterfaceProxy<ITestOutput>>();
+        result.Output.Returns(output);
+        result.Test.DisplayName.Returns(testDisplayName);
+        result.TestAssembly.Assembly.AssemblyPath.Returns(assemblyPath);
+        return result;
+    }
+
     public static ITestPassed TestPassed(Type type, string methodName, string displayName = null, string output = null, decimal executionTime = 0M)
     {
         var testCase = TestCase(type, methodName);