Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Consider inner test suites for NUnit results #331

Merged
merged 1 commit into from Jul 19, 2022
Merged

Consider inner test suites for NUnit results #331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 17 additions & 2 deletions python/publish/junit.py
View file
Open in desktop
Expand Up @@ -192,7 +192,7 @@ def int_opt(string: Optional[str]) -> Optional[int]:

def get_cases(suite: TestSuite) -> List[TestCase]:
"""
JUnit seems to allow for testsuite tags inside testsuite tags, potentially at any depth.
JUnit allows for testsuite tags inside testsuite tags at any depth.
https://llg.cubic.org/docs/junit/

This skips all inner testsuite tags and returns a list of all contained testcase tags.
Expand All @@ -203,6 +203,19 @@ def get_cases(suite: TestSuite) -> List[TestCase]:
for suite in suites
for case in get_cases(suite)] + cases

def get_suites(suite: TestSuite) -> List[TestSuite]:
"""
JUnit allows for testsuite tags inside testsuite tags at any depth.
https://llg.cubic.org/docs/junit/

This enumerates all leaf testsuite tags and those with testcases tags.
"""
suites = list(suite.iterchildren(TestSuite))
cases = list(suite.iterchildren(TestCase))
return [leaf_suite
for suite in suites
for leaf_suite in get_suites(suite)] + ([suite] if cases or not suites else [])

cases = [
UnitTestCase(
result_file=result_file,
Expand All @@ -225,7 +238,9 @@ def get_cases(suite: TestSuite) -> List[TestCase]:
files=len(list(trees)),
errors=errors,
# test state counts from suites
suites=len(suites),
suites=len([leaf_suite
for _, suite in suites
for leaf_suite in get_suites(suite)]),
suite_tests=suite_tests,
suite_skipped=suite_skipped,
suite_failures=suite_failures,
Expand Down
84 changes: 38 additions & 46 deletions python/publish/xslt/nunit3-to-junit.xslt
View file
Open in desktop
Expand Up @@ -34,52 +34,44 @@
</xsl:template>

<xsl:template match="test-suite">
<xsl:choose>
<xsl:when test="test-case or results/test-case">
<testsuite>
<xsl:if test="@testcasecount">
<xsl:attribute name="tests"><xsl:value-of select="@testcasecount"/></xsl:attribute>
</xsl:if>
<xsl:if test="@failed">
<xsl:attribute name="failures"><xsl:value-of select="@failed"/></xsl:attribute>
</xsl:if>
<xsl:if test="@errors">
<xsl:attribute name="errors"><xsl:value-of select="@errors"/></xsl:attribute>
</xsl:if>
<xsl:if test="@skipped">
<xsl:attribute name="skipped"><xsl:value-of select="@skipped"/></xsl:attribute>
</xsl:if>
<xsl:if test="@duration or @time">
<xsl:attribute name="time">
<xsl:choose>
<xsl:when test="@duration"><xsl:value-of select="@duration"/></xsl:when>
<xsl:when test="@time"><xsl:value-of select="@time"/></xsl:when>
</xsl:choose>
</xsl:attribute>
</xsl:if>
<xsl:if test="@start-time">
<xsl:attribute name="timestamp"><xsl:value-of select="@start-time"/></xsl:attribute>
</xsl:if>
<xsl:attribute name="name">
<xsl:choose>
<xsl:when test="@fullname"><xsl:value-of select="@fullname"/></xsl:when>
<xsl:when test="@classname"><xsl:value-of select="@classname"/></xsl:when>
<xsl:otherwise>
<xsl:for-each select="ancestor::test-suite[@type='TestSuite' or @type='Namespace']/@name">
<xsl:value-of select="concat(., '.')"/>
</xsl:for-each>
<xsl:value-of select="@name"/>
</xsl:otherwise>
</xsl:choose>
</xsl:attribute>
<xsl:apply-templates select="test-case | results/test-case"/>
</testsuite>
<xsl:apply-templates select="test-suite"/>
</xsl:when>
<xsl:otherwise>
<xsl:apply-templates/>
</xsl:otherwise>
</xsl:choose>
<testsuite>
<xsl:if test="@testcasecount">
<xsl:attribute name="tests"><xsl:value-of select="@testcasecount"/></xsl:attribute>
</xsl:if>
<xsl:if test="@failed">
<xsl:attribute name="failures"><xsl:value-of select="@failed"/></xsl:attribute>
</xsl:if>
<xsl:if test="@errors">
<xsl:attribute name="errors"><xsl:value-of select="@errors"/></xsl:attribute>
</xsl:if>
<xsl:if test="@skipped">
<xsl:attribute name="skipped"><xsl:value-of select="@skipped"/></xsl:attribute>
</xsl:if>
<xsl:if test="@duration or @time">
<xsl:attribute name="time">
<xsl:choose>
<xsl:when test="@duration"><xsl:value-of select="@duration"/></xsl:when>
<xsl:when test="@time"><xsl:value-of select="@time"/></xsl:when>
</xsl:choose>
</xsl:attribute>
</xsl:if>
<xsl:if test="@start-time">
<xsl:attribute name="timestamp"><xsl:value-of select="@start-time"/></xsl:attribute>
</xsl:if>
<xsl:attribute name="name">
<xsl:choose>
<xsl:when test="@fullname"><xsl:value-of select="@fullname"/></xsl:when>
<xsl:when test="@classname"><xsl:value-of select="@classname"/></xsl:when>
<xsl:otherwise>
<xsl:for-each select="ancestor::test-suite[@type='TestSuite' or @type='Namespace']/@name">
<xsl:value-of select="concat(., '.')"/>
</xsl:for-each>
<xsl:value-of select="@name"/>
</xsl:otherwise>
</xsl:choose>
</xsl:attribute>
<xsl:apply-templates/>
</testsuite>
</xsl:template>

<xsl:template match="test-case">
Expand Down
2 changes: 1 addition & 1 deletion python/test/files/junit-xml/testcase-in-testcase.results
View file
Open in desktop
@@ -1,7 +1,7 @@
publish.unittestresults.ParsedUnitTestResults(
files=1,
errors=[],
suites=1,
suites=4,
suite_tests=5,
suite_skipped=0,
suite_failures=0,
Expand Down
96 changes: 96 additions & 0 deletions python/test/files/nunit/mstest/clicketyclackety.junit-xml
View file
Open in desktop
@@ -0,0 +1,96 @@
<?xml version='1.0' encoding='utf-8'?>
<testsuite tests="23" failures="10" skipped="0" time="0.567280" timestamp="2022-07-13T23:20:35.0612744Z" name="C:\Users\USER\actions-runner\_work\MyProject\MyProject\tests\MyProject.tests.real\bin\x64\Release\net48\MyProject.Tests.Real.dll">


<testsuite tests="23" failures="10" skipped="0" time="0.558335" timestamp="2022-07-13T23:20:35.0612744Z" name="Integration">

<testsuite tests="23" failures="10" skipped="0" time="0.557669" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject">

<testsuite tests="23" failures="10" skipped="0" time="0.557418" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject">

<testsuite tests="23" failures="10" skipped="0" time="0.557367" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject.Tests">

<testsuite tests="23" failures="10" skipped="0" time="0.556978" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject.Tests.Real">

<testsuite tests="2" failures="2" skipped="0" time="0.199250" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject.Tests.Real.UserInput">

<testsuite tests="1" failures="1" skipped="0" time="0.142842" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject.Tests.Real.UserInput.BakeDrawingCommandTests">

<testcase name="BakeDrawings" classname="" status="Failed" time="0.135485">
<failure message="System.InvalidOperationException : Assert.Equals should not be used. Use Assert.AreEqual instead."> at NUnit.Framework.Assert.Equals(Object a, Object b)
at MyProject.Tests.Real.UserInput.BakeDrawingCommandTests.BakeDrawings()</failure>
</testcase>
</testsuite>
<testsuite tests="1" failures="1" skipped="0" time="0.052577" timestamp="2022-07-13T23:20:35.2174931Z" name="MyProject.Tests.Real.UserInput.ProjectInitCommandTests">

<testcase name="SilentRun" classname="" status="Failed" time="0.052338">
<failure message="System.NullReferenceException : Object reference not set to an instance of an object."> at MyProject.Tests.Real.UserInput.ProjectInitCommandTests.SilentRun()</failure>
</testcase>
</testsuite>
</testsuite>
<testsuite tests="10" failures="5" skipped="0" time="0.326239" timestamp="2022-07-13T23:20:35.2643689Z" name="MyProject.Tests.Real.FlagTests">

<testcase name="DeleteMyProjectObjectEventFlagTests" classname="" status="Passed" time="0.000201"/>
<testcase name="DiscardDrawingTests" classname="" status="Failed" time="0.004832">
<failure message="System.IO.DirectoryNotFoundException : Could not find a part of the path 'C:\Users\USER\actions-runner\_work\MyProject\MyProject\SC\f4a8fa46-245d-4cd5-88c1-80fcfbda6369'."> at System.IO.__Error.WinIOError(Int32 errorCode, String maybeFullPath)
at System.IO.FileSystemEnumerableIterator`1.CommonInit()
at System.IO.FileSystemEnumerableIterator`1..ctor(String path, String originalUserPath, String searchPattern, SearchOption searchOption, SearchResultHandler`1 resultHandler, Boolean checkHost)
at System.IO.Directory.GetFiles(String path)
at MyProject.Tests.Real.FlagTests.DiscardDrawingTests()</failure>
</testcase>
<testcase name="LoadDrawingsEventFlagTests" classname="" status="Failed" time="0.057537">
<failure message=" Expected: 3&#10; But was: 0&#10;"> at MyProject.Tests.Real.FlagTests.LoadDrawingsEventFlagTests()
</failure>

</testcase>
<testcase name="ModifyNewObjectUniqueIdEventFlagTests" classname="" status="Passed" time="0.000104"/>
<testcase name="MoveControlPointEventFlagTests" classname="" status="Passed" time="0.000112"/>
<testcase name="ResetProjectEventFlagTests" classname="" status="Failed" time="0.025094">
<failure message="System.NullReferenceException : Object reference not set to an instance of an object."> at MyProject.Tests.Real.FlagTests.ResetProjectEventFlagTests()</failure>
</testcase>
<testcase name="SetupLayersEventFlagTests" classname="" status="Failed" time="0.002310">
<failure message="om.Exceptions.DocumentException : Document should be initlised, but isn't!"> at MyProject.Runtime.Events.SetupLayers.Execute()
at MyProject.Tests.Real.FlagTests.SetupLayersEventFlagTests()</failure>
</testcase>
<testcase name="SetupPipeEventFlagTests" classname="" status="Failed" time="0.233069">
<failure message="System.NullReferenceException : Object reference not set to an instance of an object."> at MyProject.Tests.Real.FlagTests.SetupPipeEventFlagTests()</failure>
</testcase>
<testcase name="UpdateDrawingsPanelEventFlagTests" classname="" status="Passed" time="0.000363"/>
<testcase name="UpdatePropertiesPanelEventFlagTests" classname="" status="Passed" time="0.000068"/>
</testsuite>
<testsuite tests="1" failures="0" skipped="0" time="0.007232" timestamp="2022-07-13T23:20:35.5924996Z" name="MyProject.Tests.Real.NewDocumentTests">
<testcase name="EventRegisterTests" classname="" status="Passed" time="0.005957"/>
</testsuite>
<testsuite tests="1" failures="0" skipped="0" time="0.000429" timestamp="2022-07-13T23:20:35.5924996Z" name="MyProject.Tests.Real.PipeTests">
<testcase name="Test" classname="" status="Passed" time="0.000102"/>
</testsuite>
<testsuite tests="7" failures="3" skipped="0" time="0.017983" timestamp="2022-07-13T23:20:35.6081231Z" name="MyProject.Tests.Real.RuntimeTests">

<testcase name="DisplayGraphicConstants" classname="" status="Passed" time="0.000046"/>
<testcase name="DrawingConstants" classname="" status="Failed" time="0.001262">
<failure message="System.NullReferenceException : Object reference not set to an instance of an object."> at MyProject.Tests.Real.RuntimeTests.DrawingConstants()</failure>
</testcase>
<testcase name="FileConstants" classname="" status="Failed" time="0.001455">
<failure message="System.NullReferenceException : Object reference not set to an instance of an object."> at MyProject.Tests.Real.RuntimeTests.FileConstants()</failure>
</testcase>
<testcase name="ObjectConstants" classname="" status="Passed" time="0.000290"/>
<testcase name="PluginConstants" classname="" status="Failed" time="0.005593">
<failure message="System.MissingMethodException : Method not found: 'System.Object MyProject.MyProjectPlugIn.get_Instance()'."> at MyProject.Tests.Real.RuntimeTests.PluginConstants()</failure>
</testcase>
<testcase name="UIPanelConstants" classname="" status="Passed" time="0.007398"/>
<testcase name="UIPropertyConstants" classname="" status="Passed" time="0.000517"/>
</testsuite>
<testsuite tests="1" failures="0" skipped="0" time="0.000898" timestamp="2022-07-13T23:20:35.6237469Z" name="MyProject.Tests.Real.SwitchingTests">
<testcase name="Test" classname="" status="Passed" time="0.000254"/>
</testsuite>
<testsuite tests="1" failures="0" skipped="0" time="0.556762" timestamp="2022-07-13T23:20:35.0612744Z" name="MyProject.Tests.Real.Tests">
<testsuite tests="1" failures="0" skipped="0" time="0.001551" timestamp="2022-07-13T23:20:35.6237469Z" name="MyProject.Tests.Real.Tests.RuntimeConstants">
<testcase name="LoadedDrawings" classname="" status="Passed" time="0.000986"/>
</testsuite>
</testsuite>
</testsuite>
</testsuite>
</testsuite>
</testsuite>
</testsuite>
</testsuite>