Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Issue779 #866

Merged
merged 2 commits into from May 31, 2021
Merged

Issue779 #866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
41885ae
Fix for issue #671, exception stacktrace from OneTimeSetup
OsirisTerje May 29, 2021
41dabe8
Fix for issue #779
OsirisTerje May 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 8 additions & 8 deletions .runsettings
View file
Open in desktop
@@ -1,10 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<RunConfiguration>
<!-- 0 = As many processes as possible, limited by number of cores on machine, 1 = Sequential (1 process), 2-> Given number of processes up to limit by number of cores on machine-->
<MaxCpuCount>0</MaxCpuCount>
</RunConfiguration>
<RunConfiguration>
<!-- 0 = As many processes as possible, limited by number of cores on machine, 1 = Sequential (1 process), 2-> Given number of processes up to limit by number of cores on machine-->
<MaxCpuCount>0</MaxCpuCount>

</RunConfiguration>

<!--
<TestRunParameters>
Expand Down Expand Up @@ -46,9 +46,9 @@

<NUnit>
<Verbosity>5</Verbosity>
<DumpXmlTestResults>true</DumpXmlTestResults>
<DumpXmlTestDiscovery>true</DumpXmlTestDiscovery>
<DiscoveryMethod>Current</DiscoveryMethod>
<DumpXmlTestResults>true</DumpXmlTestResults>
<DumpXmlTestDiscovery>true</DumpXmlTestDiscovery>
<DiscoveryMethod>Current</DiscoveryMethod>
</NUnit>


Expand Down
4 changes: 3 additions & 1 deletion src/NUnitTestAdapter/AdapterSettings.cs
View file
Open in desktop
Expand Up @@ -111,6 +111,7 @@ public interface IAdapterSettings
int AssemblySelectLimit { get; }

bool UseNUnitFilter { get; }
bool IncludeStackTraceForSuites { get; }


void Load(IDiscoveryContext context);
Expand Down Expand Up @@ -236,6 +237,7 @@ public AdapterSettings(ITestLogger logger)
public bool SkipNonTestAssemblies { get; private set; }
public int AssemblySelectLimit { get; private set; }
public bool UseNUnitFilter { get; private set; }
public bool IncludeStackTraceForSuites { get; private set; }

public VsTestCategoryType VsTestCategoryType { get; private set; } = VsTestCategoryType.NUnit;

Expand Down Expand Up @@ -322,7 +324,7 @@ public void Load(string settingsXml)
ConsoleOut = GetInnerTextAsInt(nunitNode, nameof(ConsoleOut), 1); // 0 no output to console, 1 : output to console
StopOnError = GetInnerTextAsBool(nunitNode, nameof(StopOnError), false);
UseNUnitFilter = GetInnerTextAsBool(nunitNode, nameof(UseNUnitFilter), true);

IncludeStackTraceForSuites = GetInnerTextAsBool(nunitNode, nameof(IncludeStackTraceForSuites), true);

// Engine settings
DiscoveryMethod = MapEnum(GetInnerText(nunitNode, nameof(DiscoveryMethod), Verbosity > 0), DiscoveryMethod.Current);
Expand Down
6 changes: 5 additions & 1 deletion src/NUnitTestAdapter/NUnitEngine/NUnitTestEventSuiteFinished.cs
View file
Open in desktop
Expand Up @@ -8,6 +8,7 @@ public interface INUnitTestEventSuiteFinished : INUnitTestEvent
bool HasReason { get; }
string FailureMessage { get; }
bool HasFailure { get; }
string StackTrace { get; }
}

/// <summary>
Expand All @@ -28,14 +29,17 @@ public NUnitTestEventSuiteFinished(XmlNode node) : base(node)
if (failureNode != null)
{
FailureMessage = failureNode.SelectSingleNode("message")?.InnerText;
StackTrace = failureNode.SelectSingleNode("stack-trace")?.InnerText;
}
ReasonMessage = Node.SelectSingleNode("reason/message")?.InnerText;
}

public string ReasonMessage { get; }

public bool HasReason => !string.IsNullOrEmpty(ReasonMessage);
public string FailureMessage { get; }
public string FailureMessage { get; } = "";

public string StackTrace { get; } = "";

public bool HasFailure => !string.IsNullOrEmpty(FailureMessage);
}
Expand Down
74 changes: 39 additions & 35 deletions src/NUnitTestAdapter/NUnitEventListener.cs
View file
Open in desktop
Expand Up @@ -47,10 +47,10 @@ public class NUnitEventListener :
ITestEventListener, IDisposable // Public for testing
{
private static readonly ICollection<INUnitTestEventTestOutput> EmptyNodes = new List<INUnitTestEventTestOutput>();
private readonly ITestExecutionRecorder _recorder;
private readonly ITestConverterCommon _testConverter;
private readonly IAdapterSettings _settings;
private readonly Dictionary<string, ICollection<INUnitTestEventTestOutput>> _outputNodes = new ();
private readonly ITestExecutionRecorder recorder;
private readonly ITestConverterCommon testConverter;
private readonly IAdapterSettings settings;
private readonly Dictionary<string, ICollection<INUnitTestEventTestOutput>> outputNodes = new ();

#if NET35
public override object InitializeLifetimeService()
Expand All @@ -69,9 +69,9 @@ public NUnitEventListener(ITestConverterCommon testConverter, INUnit3TestExecuto
{
this.executor = executor;
dumpXml = executor.Dump;
_settings = executor.Settings;
_recorder = executor.FrameworkHandle;
_testConverter = testConverter;
settings = executor.Settings;
recorder = executor.FrameworkHandle;
this.testConverter = testConverter;
}

#region ITestEventListener
Expand Down Expand Up @@ -103,8 +103,8 @@ public void OnTestEvent(string report)
}
catch (Exception ex)
{
_recorder.SendMessage(TestMessageLevel.Warning, $"Error processing {node.Name} event for {node.FullName}");
_recorder.SendMessage(TestMessageLevel.Warning, ex.ToString());
recorder.SendMessage(TestMessageLevel.Warning, $"Error processing {node.Name} event for {node.FullName}");
recorder.SendMessage(TestMessageLevel.Warning, ex.ToString());
}
}

Expand Down Expand Up @@ -141,46 +141,49 @@ protected virtual void Dispose(bool disposing)

public void TestStarted(INUnitTestEventStartTest testNode)
{
var ourCase = _testConverter.GetCachedTestCase(testNode.Id);
var ourCase = testConverter.GetCachedTestCase(testNode.Id);

// Simply ignore any TestCase not found in the cache
if (ourCase != null)
_recorder.RecordStart(ourCase);
recorder.RecordStart(ourCase);
}

public void TestFinished(INUnitTestEventTestCase resultNode)
{
var testId = resultNode.Id;
if (_outputNodes.TryGetValue(testId, out var outputNodes))
if (this.outputNodes.TryGetValue(testId, out var outputNodes))
{
_outputNodes.Remove(testId);
this.outputNodes.Remove(testId);
}

var result = _testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes);
if (_settings.ConsoleOut == 1)
var result = testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes);
if (settings.ConsoleOut == 1)
{
if (!string.IsNullOrEmpty(result.ConsoleOutput) && result.ConsoleOutput != NL)
{
string msg = result.ConsoleOutput;
if (_settings.UseTestNameInConsoleOutput)
if (settings.UseTestNameInConsoleOutput)
msg = $"{resultNode.Name}: {msg}";
_recorder.SendMessage(TestMessageLevel.Informational, msg);
recorder.SendMessage(TestMessageLevel.Informational, msg);
}
if (!string.IsNullOrEmpty(resultNode.ReasonMessage))
{
_recorder.SendMessage(TestMessageLevel.Informational, $"{resultNode.Name}: {resultNode.ReasonMessage}");
recorder.SendMessage(TestMessageLevel.Informational, $"{resultNode.Name}: {resultNode.ReasonMessage}");
}
}

_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
if (result.TestCaseResult != null)
{
_recorder.RecordResult(vsResult);
}
recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
recorder.RecordResult(vsResult);
}

if (result.TestCaseResult.Outcome == TestOutcome.Failed && _settings.StopOnError)
{
executor.StopRun();
if (result.TestCaseResult.Outcome == TestOutcome.Failed && settings.StopOnError)
{
executor.StopRun();
}
}
}

Expand All @@ -191,15 +194,16 @@ public void SuiteFinished(INUnitTestEventSuiteFinished resultNode)
var site = resultNode.Site();
if (site != NUnitTestEvent.SiteType.Setup && site != NUnitTestEvent.SiteType.TearDown)
return;
_recorder.SendMessage(TestMessageLevel.Warning, $"{site} failed for test fixture {resultNode.FullName}");
recorder.SendMessage(TestMessageLevel.Warning, $"{site} failed for test fixture {resultNode.FullName}");

if (resultNode.HasFailure)
_recorder.SendMessage(TestMessageLevel.Warning, resultNode.FailureMessage);

// Should not be any stacktrace on Suite-finished
// var stackNode = resultNode.Failure.StackTrace;
// if (!string.IsNullOrEmpty(stackNode))
// _recorder.SendMessage(TestMessageLevel.Warning, stackNode);
{
string msg = resultNode.FailureMessage;
var stackNode = resultNode.StackTrace;
if (!string.IsNullOrEmpty(stackNode) && settings.IncludeStackTraceForSuites)
msg += $"\nStackTrace: {stackNode}";
recorder.SendMessage(TestMessageLevel.Warning, msg);
}
}

private static readonly string NL = Environment.NewLine;
Expand All @@ -222,16 +226,16 @@ public void TestOutput(INUnitTestEventTestOutput outputNodeEvent)
string testId = outputNodeEvent.TestId;
if (!string.IsNullOrEmpty(testId))
{
if (!_outputNodes.TryGetValue(testId, out var outputNodes))
if (!this.outputNodes.TryGetValue(testId, out var outputNodes))
{
outputNodes = new List<INUnitTestEventTestOutput>();
_outputNodes.Add(testId, outputNodes);
this.outputNodes.Add(testId, outputNodes);
}

outputNodes.Add(outputNodeEvent);
}

_recorder.SendMessage(TestMessageLevel.Warning, text);
recorder.SendMessage(TestMessageLevel.Warning, text);
}
}
}
24 changes: 11 additions & 13 deletions src/NUnitTestAdapter/TestFilterConverter/TestFilterParser.cs
View file
Open in desktop
Expand Up @@ -85,7 +85,7 @@ public string ParseFilterExpression()

var sb = new StringBuilder("<or>");

foreach (string term in terms)
foreach (var term in terms)
sb.Append(term);

sb.Append("</or>");
Expand All @@ -111,7 +111,7 @@ public string ParseFilterTerm()

var sb = new StringBuilder("<and>");

foreach (string element in elements)
foreach (var element in elements)
sb.Append(element);

sb.Append("</and>");
Expand All @@ -128,12 +128,12 @@ public string ParseFilterCondition()
if (LookingAt(LPAREN, NOT_OP))
return ParseExpressionInParentheses();

Token lhs = Expect(TokenKind.Word);
var lhs = Expect(TokenKind.Word);

if (!LookingAt(REL_OPS))
return EmitFullNameFilter(CONTAINS_OP, lhs.Text);

Token op = Expect(REL_OPS);
var op = Expect(REL_OPS);
Token rhs;

switch (lhs.Text)
Expand All @@ -153,10 +153,8 @@ public string ParseFilterCondition()

default:
// Assume it's a property name
// op = Expect(REL_OPS);
// rhs = Expect(TokenKind.String, TokenKind.Word);
// return EmitPropertyElement(op, lhs, rhs);
throw InvalidTokenError(lhs);
rhs = Expect(TokenKind.String, TokenKind.Word);
return EmitPropertyFilter(op, lhs.Text, rhs.Text);
}
}

Expand Down Expand Up @@ -217,11 +215,11 @@ private static string EscapeRhsValue(Token op, string rhs)

private string ParseExpressionInParentheses()
{
Token op = Expect(LPAREN, NOT_OP);
var op = Expect(LPAREN, NOT_OP);

if (op == NOT_OP) Expect(LPAREN);

string result = ParseFilterExpression();
var result = ParseFilterExpression();

Expect(RPAREN);

Expand All @@ -234,7 +232,7 @@ private string ParseExpressionInParentheses()
// Require a token of one or more kinds
private Token Expect(params TokenKind[] kinds)
{
Token token = NextToken();
var token = NextToken();

if (kinds.Any(kind => token.Kind == kind))
{
Expand All @@ -247,7 +245,7 @@ private Token Expect(params TokenKind[] kinds)
// Require a token from a list of tokens
private Token Expect(params Token[] valid)
{
Token token = NextToken();
var token = NextToken();

if (valid.Any(item => token == item))
{
Expand Down Expand Up @@ -284,7 +282,7 @@ private static string EscapeRegexChars(string input)
{
var sb = new StringBuilder();

foreach (char c in input)
foreach (var c in input)
{
if (REGEX_CHARS.Contains(c))
sb.Append('\\');
Expand Down
2 changes: 1 addition & 1 deletion src/NUnitTestAdapter/TestFilterConverter/Tokenizer.cs
View file
Open in desktop
Expand Up @@ -31,7 +31,7 @@ public enum TokenKind
Eof,
Word,
FQN,
// String, // Unused
String, // Unused
Symbol
}

Expand Down
30 changes: 30 additions & 0 deletions src/NUnitTestAdapterTests/NUnitEngineTests/NUnitTestEventsTests.cs
View file
Open in desktop
Expand Up @@ -223,6 +223,36 @@ public void ThatTestEventIsParsedForFinishSuiteWithFailure()
Assert.That(sut.FailureMessage, Is.EqualTo("One or more child tests had errors"));
}

/// <summary>
/// Exception in OneTimeSetUp
/// </summary>
private readonly string testSuiteFinishedWithFailureAndStackTrace = @"<test-suite type='TestFixture' id='0-1000' name='Tests' fullname='Issue671.Tests' classname='Issue671.Tests' runstate='Runnable' testcasecount='2' result='Failed' label='Error' site='SetUp' start-time='2021-05-29T09:34:10.2386869Z' end-time='2021-05-29T09:34:10.2532435Z' duration='0.014557' total='2' passed='0' failed='2' warnings='0' inconclusive='0' skipped='0' asserts='0'>
<failure>
<message><![CDATA[System.Exception : oops Deep Down]]></message>
<stack-trace><![CDATA[ at Issue671.SomeWhereDeepDown.WhatDoWeDoHere() in d:\repos\NUnit\nunit3-vs-adapter.issues\Issue671\UnitTest1.cs:line 44
at Issue671.Tests.OneTimeSetup() in d:\repos\NUnit\nunit3-vs-adapter.issues\Issue671\UnitTest1.cs:line 12]]></stack-trace>
</failure>
<test-case id='0-1001' name='Test1' fullname='Issue671.Tests.Test1' methodname='Test1' classname='Issue671.Tests' runstate='Runnable' seed='1958545482' result='Failed' label='Error' site='Parent' start-time='2021-05-29T09:34:10.2386869Z' end-time='2021-05-29T09:34:10.2510075Z' duration='0.012408' asserts='0'>
<failure>
<message><![CDATA[OneTimeSetUp: System.Exception : oops Deep Down]]></message>
</failure>
</test-case>
<test-case id='0-1002' name='Test2' fullname='Issue671.Tests.Test2' methodname='Test2' classname='Issue671.Tests' runstate='Runnable' seed='711019934' result='Failed' label='Error' site='Parent' start-time='2021-05-29T09:34:10.2386869Z' end-time='2021-05-29T09:34:10.2523634Z' duration='0.013677' asserts='0'>
<failure>
<message><![CDATA[OneTimeSetUp: System.Exception : oops Deep Down]]></message>
</failure>
</test-case>
</test-suite>";

[Test]
public void ThatTestEventIsParsedForFinishSuiteWithExceptionInOneTimeSetUp()
{
var sut = new NUnitTestEventSuiteFinished(testSuiteFinishedWithFailureAndStackTrace);
Assert.That(sut.HasFailure);
Assert.That(sut.FailureMessage.Length, Is.GreaterThan(0));
Assert.That(sut.StackTrace.Length, Is.GreaterThan(0));
}



private readonly string testCaseSucceedsWithOutput = @"<test-case id='0-1074' name='TestSucceeds' fullname='NUnitTestDemo.SimpleTests.TestSucceeds' methodname='TestSucceeds' classname='NUnitTestDemo.SimpleTests' runstate='Runnable' seed='1232497275' result='Passed' start-time='2020-01-24 11:18:32Z' end-time='2020-01-24 11:18:32Z' duration='0.016868' asserts='1' parentId='0-1073'>
Expand Down
1 change: 1 addition & 0 deletions src/NUnitTestAdapterTests/TestFilterConverterTests/TestFilterParserTests.cs
View file
Open in desktop
Expand Up @@ -168,6 +168,7 @@ public void CreateParser()
[TestCase(
"TestCategory=Urgent & !(FullyQualifiedName=My.Tests | TestCategory = high)",
"<and><cat>Urgent</cat><not><or><test>My.Tests</test><cat>high</cat></or></not></and>")]
[TestCase("Bug = 12345", "<prop name='Bug'>12345</prop>")]
public void TestParser(string input, string output)
{
Assert.That(_parser.Parse(input), Is.EqualTo($"<filter>{output}</filter>"));
Expand Down