Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix class/assembly cleanups log collect and attachment #1470

Merged
merged 2 commits into from
Dec 16, 2022
Merged

Fix class/assembly cleanups log collect and attachment #1470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c10eaa
Fix class/assembly cleanups log collect and attachment
Evangelink Dec 14, 2022
87755bb
Fix remaining tests count mechanism
Evangelink Dec 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
54 changes: 54 additions & 0 deletions src/Adapter/MSTest.TestAdapter/Execution/TestAssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,4 +234,58 @@ public void RunAssemblyInitialize(TestContext testContext)
}
}
}

/// <summary>
/// Calls the assembly cleanup method in a thread-safe.
/// </summary>
/// <remarks>
/// It is a replacement for RunAssemblyCleanup but as we are in a bug-fix version, we do not want to touch
/// public API and so we introduced this method.
/// </remarks>
internal void ExecuteAssemblyCleanup()
Evangelink marked this conversation as resolved.
Show resolved Hide resolved
{
if (AssemblyCleanupMethod == null)
MarcoRossignoli marked this conversation as resolved.
Show resolved Hide resolved
Evangelink marked this conversation as resolved.
Show resolved Hide resolved
{
return;
}

lock (_assemblyInfoExecuteSyncObject)
{
try
{
AssemblyCleanupMethod.InvokeAsSynchronousTask(null);
}
catch (Exception ex)
{
var realException = ex.InnerException ?? ex;

string errorMessage;

// special case AssertFailedException to trim off part of the stack trace
if (realException is AssertFailedException or AssertInconclusiveException)
{
errorMessage = realException.Message;
}
else
{
errorMessage = StackTraceHelper.GetExceptionMessage(realException);
}

var exceptionStackTraceInfo = StackTraceHelper.GetStackTraceInformation(realException);
DebugEx.Assert(AssemblyCleanupMethod.DeclaringType?.Name is not null, "AssemblyCleanupMethod.DeclaringType.Name is null");

throw new TestFailedException(
UnitTestOutcome.Failed,
string.Format(
CultureInfo.CurrentCulture,
Resource.UTA_AssemblyCleanupMethodWasUnsuccesful,
AssemblyCleanupMethod.DeclaringType.Name,
AssemblyCleanupMethod.Name,
errorMessage,
exceptionStackTraceInfo?.ErrorStackTrace),
exceptionStackTraceInfo,
realException);
}
}
}
}
70 changes: 70 additions & 0 deletions src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,4 +410,74 @@ public void RunClassInitialize(TestContext testContext)

return null;
}

/// <summary>
/// Execute current and base class cleanups.
/// </summary>
/// <remarks>
/// This is a replacement for RunClassCleanup but as we are on a bug fix version, we do not want to change
/// the public API, hence this method.
/// </remarks>
internal void ExecuteClassCleanup()
Evangelink marked this conversation as resolved.
Show resolved Hide resolved
{
if ((ClassCleanupMethod is null && BaseClassInitAndCleanupMethods.All(p => p.Item2 == null))
|| IsClassCleanupExecuted)
{
return;
}

lock (_testClassExecuteSyncObject)
{
if (IsClassCleanupExecuted
|| (!IsClassInitializeExecuted && ClassInitializeMethod is not null))
{
return;
}

MethodInfo? classCleanupMethod = null;

try
{
classCleanupMethod = ClassCleanupMethod;
classCleanupMethod?.InvokeAsSynchronousTask(null);
var baseClassCleanupQueue = new Queue<MethodInfo>(BaseClassCleanupMethodsStack);
while (baseClassCleanupQueue.Count > 0)
{
classCleanupMethod = baseClassCleanupQueue.Dequeue();
classCleanupMethod?.InvokeAsSynchronousTask(null);
}

IsClassCleanupExecuted = true;

return;
}
catch (Exception exception)
{
var realException = exception.InnerException ?? exception;
ClassCleanupException = realException;

// special case AssertFailedException to trim off part of the stack trace
string errorMessage = realException is AssertFailedException or AssertInconclusiveException
? realException.Message
: StackTraceHelper.GetExceptionMessage(realException);

var exceptionStackTraceInfo = realException.TryGetStackTraceInformation();

var testFailedException = new TestFailedException(
ObjectModelUnitTestOutcome.Failed,
string.Format(
CultureInfo.CurrentCulture,
Resource.UTA_ClassCleanupMethodWasUnsuccesful,
classCleanupMethod!.DeclaringType!.Name,
classCleanupMethod.Name,
errorMessage,
exceptionStackTraceInfo?.ErrorStackTrace),
exceptionStackTraceInfo,
realException);
ClassCleanupException = testFailedException;

throw testFailedException;
}
}
}
}
179 changes: 48 additions & 131 deletions src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,15 +141,15 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<strin
}

DebugEx.Assert(testMethodInfo is not null, "testMethodInfo should not be null.");
RunCleanupsIfNeeded(testContext, testMethodInfo, testMethod, notRunnableResult);
RunRequiredCleanups(testContext, testMethodInfo, testMethod, notRunnableResult);

return notRunnableResult;
}

DebugEx.Assert(testMethodInfo is not null, "testMethodInfo should not be null.");
var testMethodRunner = new TestMethodRunner(testMethodInfo, testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces);
var result = testMethodRunner.Execute();
RunCleanupsIfNeeded(testContext, testMethodInfo, testMethod, result);
RunRequiredCleanups(testContext, testMethodInfo, testMethod, result);
return result;
}
catch (TypeInspectionException ex)
Expand All @@ -159,108 +159,60 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<strin
}
}

/// <summary>
/// Runs the class cleanup method.
/// It returns any error information during the execution of the cleanup method.
/// </summary>
/// <returns> The <see cref="RunCleanupResult"/>. </returns>
internal RunCleanupResult? RunClassAndAssemblyCleanup()
{
// No cleanup methods to execute, then return.
var assemblyInfoCache = _typeCache.AssemblyInfoListWithExecutableCleanupMethods;
var classInfoCache = _typeCache.ClassInfoListWithExecutableCleanupMethods;
if (assemblyInfoCache.Length == 0 && classInfoCache.Length == 0)
{
return null;
}

var result = new RunCleanupResult { Warnings = new List<string>() };

using (var redirector = new LogMessageListener(MSTestSettings.CurrentSettings.CaptureDebugTraces))
{
try
{
RunClassCleanupMethods(classInfoCache, result.Warnings);
RunAssemblyCleanup(assemblyInfoCache, result.Warnings);
}
finally
{
// Replacing the null character with a string.replace should work.
// If this does not work for a specific dotnet version a custom function doing the same needs to be put in place.
result.StandardOut = redirector.GetAndClearStandardOutput()?.Replace("\0", "\\0");
result.StandardError = redirector.GetAndClearStandardError()?.Replace("\0", "\\0");
result.DebugTrace = redirector.GetAndClearDebugTrace()?.Replace("\0", "\\0");
}
}

return result;
}

private void RunCleanupsIfNeeded(ITestContext testContext, TestMethodInfo testMethodInfo, TestMethod testMethod, UnitTestResult[] results)
private void RunRequiredCleanups(ITestContext testContext, TestMethodInfo testMethodInfo, TestMethod testMethod, UnitTestResult[] results)
{
bool shouldRunClassCleanup = false;
bool shouldRunClassAndAssemblyCleanup = false;
_classCleanupManager?.MarkTestComplete(testMethodInfo, testMethod, out shouldRunClassCleanup, out shouldRunClassAndAssemblyCleanup);

using LogMessageListener logListener = new(MSTestSettings.CurrentSettings.CaptureDebugTraces);
try
{
using LogMessageListener logListener = new(MSTestSettings.CurrentSettings.CaptureDebugTraces);
if (shouldRunClassCleanup)
{
try
{
// Class cleanup can throw exceptions in which case we need to ensure that we fail the test.
testMethodInfo.Parent.RunClassCleanup(ClassCleanupBehavior.EndOfClass);
}
finally
{
string cleanupLogs = logListener.StandardOutput;
string? cleanupTrace = logListener.DebugTrace;
string cleanupErrorLogs = logListener.StandardError;
var cleanupTestContextMessages = testContext.GetAndClearDiagnosticMessages();

if (results.Length > 0)
{
var lastResult = results[results.Length - 1];
lastResult.StandardOut += cleanupLogs;
lastResult.StandardError += cleanupErrorLogs;
lastResult.DebugTrace += cleanupTrace;
lastResult.TestContextMessages += cleanupTestContextMessages;
}
}
testMethodInfo.Parent.ExecuteClassCleanup();
}

if (shouldRunClassAndAssemblyCleanup)
{
try
var classInfoCache = _typeCache.ClassInfoListWithExecutableCleanupMethods;
foreach (var classInfo in classInfoCache)
{
RunClassAndAssemblyCleanup();
classInfo.ExecuteClassCleanup();
}
finally
{
string cleanupLogs = logListener.StandardOutput;
string? cleanupTrace = logListener.DebugTrace;
string cleanupErrorLogs = logListener.StandardError;
var cleanupTestContextMessages = testContext.GetAndClearDiagnosticMessages();

if (results.Length > 0)
{
var lastResult = results[results.Length - 1];
lastResult.StandardOut += cleanupLogs;
lastResult.StandardError += cleanupErrorLogs;
lastResult.DebugTrace += cleanupTrace;
lastResult.TestContextMessages += cleanupTestContextMessages;
}
var assemblyInfoCache = _typeCache.AssemblyInfoListWithExecutableCleanupMethods;
foreach (var assemblyInfo in assemblyInfoCache)
{
assemblyInfo.ExecuteAssemblyCleanup();
}
}
}
catch (Exception e)
catch (Exception ex)
{
// We mainly expect TestFailedException here as each cleanup method is executed in a try-catch block but
// for the sake of the catch-all mechanism, let's keep it as Exception.
if (results.Length == 0)
{
return;
}

var lastResult = results[results.Length - 1];
lastResult.Outcome = ObjectModel.UnitTestOutcome.Failed;
lastResult.ErrorMessage = ex.Message;
lastResult.ErrorStackTrace = ex.StackTrace;
}
finally
{
var cleanupTestContextMessages = testContext.GetAndClearDiagnosticMessages();

if (results.Length > 0)
{
results[results.Length - 1].Outcome = ObjectModel.UnitTestOutcome.Failed;
results[results.Length - 1].ErrorMessage = e.Message;
results[results.Length - 1].ErrorStackTrace = e.StackTrace;
var lastResult = results[results.Length - 1];
lastResult.StandardOut += logListener.StandardOutput;
lastResult.StandardError += logListener.StandardError;
lastResult.DebugTrace += logListener.DebugTrace;
lastResult.TestContextMessages += cleanupTestContextMessages;
}
}
}
Expand Down Expand Up @@ -331,44 +283,6 @@ private void RunCleanupsIfNeeded(ITestContext testContext, TestMethodInfo testMe
return true;
}

/// <summary>
/// Run assembly cleanup methods.
/// </summary>
/// <param name="assemblyInfoCache"> The assembly Info Cache. </param>
/// <param name="warnings"> The warnings. </param>
private static void RunAssemblyCleanup(IEnumerable<TestAssemblyInfo> assemblyInfoCache, IList<string> warnings)
{
foreach (var assemblyInfo in assemblyInfoCache)
{
DebugEx.Assert(assemblyInfo.HasExecutableCleanupMethod, "HasExecutableCleanupMethod should be true.");

var warning = assemblyInfo.RunAssemblyCleanup();
if (warning != null)
{
warnings.Add(warning);
}
}
}

/// <summary>
/// Run class cleanup methods.
/// </summary>
/// <param name="classInfoCache"> The class Info Cache. </param>
/// <param name="warnings"> The warnings. </param>
private static void RunClassCleanupMethods(IEnumerable<TestClassInfo> classInfoCache, IList<string> warnings)
{
foreach (var classInfo in classInfoCache)
{
DebugEx.Assert(classInfo.HasExecutableCleanupMethod, "HasExecutableCleanupMethod should be true.");

var warning = classInfo.RunClassCleanup();
if (warning != null)
{
warnings.Add(warning);
}
}
}

private class ClassCleanupManager
{
private readonly ClassCleanupBehavior? _lifecycleFromMsTest;
Expand All @@ -385,30 +299,33 @@ private class ClassCleanupManager
_remainingTestsByClass = testsToRun.GroupBy(t => t.TestMethod.FullClassName)
.ToDictionary(
g => g.Key,
g => new HashSet<string>(g.Select(t => t.DisplayName!)));
g => new HashSet<string>(g.Select(t => t.TestMethod.UniqueName)));
_lifecycleFromMsTest = lifecycleFromMsTest;
_lifecycleFromAssembly = lifecycleFromAssembly;
_reflectHelper = reflectHelper ?? new ReflectHelper();
}

public void MarkTestComplete(TestMethodInfo testMethodInfo, TestMethod testMethod, out bool shouldRunClassCleanup, out bool shouldRunAssemblyCleanup)
public void MarkTestComplete(TestMethodInfo testMethodInfo, TestMethod testMethod, out bool shouldRunEndOfClassCleanup, out bool shouldRunEndOfAssemblyCleanup)
{
shouldRunClassCleanup = false;
shouldRunEndOfClassCleanup = false;
var testsByClass = _remainingTestsByClass[testMethodInfo.TestClassName];
lock (testsByClass)
{
testsByClass.Remove(testMethod.DisplayName!);
if (testsByClass.Count == 0 && testMethodInfo.Parent.HasExecutableCleanupMethod)
testsByClass.Remove(testMethod.UniqueName);
if (testsByClass.Count == 0)
{
var cleanupLifecycle = _reflectHelper.GetClassCleanupBehavior(testMethodInfo.Parent)
?? _lifecycleFromMsTest
?? _lifecycleFromAssembly;

shouldRunClassCleanup = cleanupLifecycle == ClassCleanupBehavior.EndOfClass;
_remainingTestsByClass.Remove(testMethodInfo.TestClassName);
if (testMethodInfo.Parent.HasExecutableCleanupMethod)
{
var cleanupLifecycle = _reflectHelper.GetClassCleanupBehavior(testMethodInfo.Parent)
?? _lifecycleFromMsTest
?? _lifecycleFromAssembly;

shouldRunEndOfClassCleanup = cleanupLifecycle == ClassCleanupBehavior.EndOfClass;
}
}

shouldRunAssemblyCleanup = _remainingTestsByClass.Count == 0;
shouldRunEndOfAssemblyCleanup = _remainingTestsByClass.Count == 0;
}
}
}
Expand Down