From 18ad368986a95a78aa2631689a381ac1d02e81cf Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Mon, 1 Feb 2021 19:30:32 +0100 Subject: [PATCH] Fix container failure XML reporting (#2542) Prior to this commit, failing containers were only reported in case they contained at least one test. However, for example for a parameterized Jupiter tests and an exception in a `@BeforeAll` method, that led to failures being silently swallowed. Now, in addition to tests, leaf nodes of the test tree are always included in the XML report, even if they are containers, not tests. Moreover, failures on the container level that occurred after their children had been executed were not reported, e.g. when an exception was thrown from a Jupiter `@AfterAll` method. Now such failures cause the contained tests to be reported as failed. Fixes #2537. --- .../release-notes/release-notes-5.7.1.adoc | 2 + .../DemoEngineExecutionContext.java | 2 +- .../DemoHierarchicalTestDescriptor.java | 2 +- .../DemoHierarchicalTestEngine.java | 23 ++- .../reporting/legacy/xml/XmlReportData.java | 38 ++-- .../reporting/legacy/xml/XmlReportWriter.java | 172 ++++++++++-------- ...egacyXmlReportGeneratingListenerTests.java | 90 +++++++-- .../legacy/xml/XmlReportDataTests.java | 18 +- 8 files changed, 221 insertions(+), 126 deletions(-) diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc index 34aec9be9e00..535dd7bd657f 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc @@ -27,6 +27,8 @@ GitHub. of this change may be witnessed by end users and test engine or extension authors. * Method `scanForClassesInPackage(String)` in `ClasspathScanner` now returns a valid list of class names when the package name is equal to the name of a module on the module path. +* The legacy XML report now always includes container-level failures (e.g. from + `@BeforeAll` Jupiter lifecycle methods). ==== Deprecations and Breaking Changes diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java index 174d51196bcf..8baf48abf067 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java @@ -13,5 +13,5 @@ /** * @since 1.0 */ -class DemoEngineExecutionContext implements EngineExecutionContext { +public class DemoEngineExecutionContext implements EngineExecutionContext { } diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java index 57058e827a75..320e712a7d28 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java @@ -26,7 +26,7 @@ public class DemoHierarchicalTestDescriptor extends AbstractTestDescriptor imple private String skippedReason; private boolean skipped; - DemoHierarchicalTestDescriptor(UniqueId uniqueId, String displayName, Runnable executeBlock) { + public DemoHierarchicalTestDescriptor(UniqueId uniqueId, String displayName, Runnable executeBlock) { this(uniqueId, displayName, null, executeBlock); } diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java index f389c7682690..f4e6c3fd300c 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java @@ -10,6 +10,8 @@ package org.junit.platform.engine.support.hierarchical; +import java.util.function.Function; + import org.junit.platform.engine.EngineDiscoveryRequest; import org.junit.platform.engine.ExecutionRequest; import org.junit.platform.engine.TestDescriptor; @@ -47,10 +49,8 @@ public DemoHierarchicalTestDescriptor addTest(String uniqueName, Runnable execut } public DemoHierarchicalTestDescriptor addTest(String uniqueName, String displayName, Runnable executeBlock) { - var uniqueId = engineDescriptor.getUniqueId().append("test", uniqueName); - var child = new DemoHierarchicalTestDescriptor(uniqueId, displayName, executeBlock); - engineDescriptor.addChild(child); - return child; + return addChild(uniqueName, uniqueId -> new DemoHierarchicalTestDescriptor(uniqueId, displayName, executeBlock), + "test"); } public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, String displayName, TestSource source) { @@ -64,10 +64,17 @@ public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, Runna public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, String displayName, TestSource source, Runnable beforeBlock) { - var uniqueId = engineDescriptor.getUniqueId().append("container", uniqueName); - var container = new DemoHierarchicalContainerDescriptor(uniqueId, displayName, source, beforeBlock); - engineDescriptor.addChild(container); - return container; + return addChild(uniqueName, + uniqueId -> new DemoHierarchicalContainerDescriptor(uniqueId, displayName, source, beforeBlock), + "container"); + } + + public > T addChild(String uniqueName, + Function creator, String segmentType) { + var uniqueId = engineDescriptor.getUniqueId().append(segmentType, uniqueName); + var child = creator.apply(uniqueId); + engineDescriptor.addChild(child); + return child; } @Override diff --git a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java index 000fbd7015c7..18437a60b6b5 100644 --- a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java +++ b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java @@ -11,8 +11,8 @@ package org.junit.platform.reporting.legacy.xml; import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; import static org.junit.platform.engine.TestExecutionResult.Status.ABORTED; -import static org.junit.platform.engine.TestExecutionResult.Status.SUCCESSFUL; import java.time.Clock; import java.time.Duration; @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; @@ -103,19 +104,11 @@ String getSkipReason(TestIdentifier testIdentifier) { }).orElse(null); } - Optional getResult(TestIdentifier testIdentifier) { - if (this.finishedTests.containsKey(testIdentifier)) { - return Optional.of(this.finishedTests.get(testIdentifier)); - } - Optional parent = this.testPlan.getParent(testIdentifier); - Optional ancestor = findAncestor(parent, this.finishedTests::containsKey); - if (ancestor.isPresent()) { - TestExecutionResult result = this.finishedTests.get(ancestor.get()); - if (result.getStatus() != SUCCESSFUL) { - return Optional.of(result); - } - } - return Optional.empty(); + List getResults(TestIdentifier testIdentifier) { + return getAncestors(testIdentifier).stream() // + .map(this.finishedTests::get) // + .filter(Objects::nonNull) // + .collect(toList()); } List getReportEntries(TestIdentifier testIdentifier) { @@ -123,12 +116,11 @@ List getReportEntries(TestIdentifier testIdentifier) { } private Optional findSkippedAncestor(TestIdentifier testIdentifier) { - return findAncestor(Optional.of(testIdentifier), this.skippedTests::containsKey); + return findAncestor(testIdentifier, this.skippedTests::containsKey); } - private Optional findAncestor(Optional testIdentifier, - Predicate predicate) { - Optional current = testIdentifier; + private Optional findAncestor(TestIdentifier testIdentifier, Predicate predicate) { + Optional current = Optional.of(testIdentifier); while (current.isPresent()) { if (predicate.test(current.get())) { return current; @@ -138,4 +130,14 @@ private Optional findAncestor(Optional testIdent return Optional.empty(); } + private List getAncestors(TestIdentifier testIdentifier) { + TestIdentifier current = testIdentifier; + List ancestors = new ArrayList<>(); + while (current != null) { + ancestors.add(current); + current = this.testPlan.getParent(current).orElse(null); + } + return ancestors; + } + } diff --git a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java index 4724a36948e6..015cfe0fa2d2 100644 --- a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java +++ b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java @@ -12,12 +12,23 @@ import static java.text.MessageFormat.format; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME; +import static java.util.Collections.emptyList; +import static java.util.Comparator.naturalOrder; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import static org.junit.platform.commons.util.ExceptionUtils.readStackTrace; import static org.junit.platform.commons.util.StringUtils.isNotBlank; import static org.junit.platform.engine.TestExecutionResult.Status.FAILED; import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY; import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.ERROR; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.FAILURE; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.SKIPPED; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.SUCCESS; import java.io.Writer; import java.net.InetAddress; @@ -25,10 +36,13 @@ import java.text.NumberFormat; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Properties; import java.util.TreeSet; @@ -41,7 +55,9 @@ import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; import org.junit.platform.reporting.legacy.LegacyReportingUtils; +import org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type; /** * {@code XmlReportWriter} writes an XML report whose format is compatible @@ -62,18 +78,28 @@ class XmlReportWriter { this.reportData = reportData; } - void writeXmlReport(TestIdentifier testIdentifier, Writer out) throws XMLStreamException { - // @formatter:off - List tests = this.reportData.getTestPlan().getDescendants(testIdentifier) - .stream() - .filter(TestIdentifier::isTest) - .collect(toList()); - // @formatter:on - writeXmlReport(testIdentifier, tests, out); + void writeXmlReport(TestIdentifier rootDescriptor, Writer out) throws XMLStreamException { + TestPlan testPlan = this.reportData.getTestPlan(); + Map tests = testPlan.getDescendants(rootDescriptor) // + .stream() // + .filter(testIdentifier -> shouldInclude(testPlan, testIdentifier)) // + .collect(toMap(identity(), this::toAggregatedResult)); // + writeXmlReport(rootDescriptor, tests, out); } - private void writeXmlReport(TestIdentifier testIdentifier, List tests, Writer out) - throws XMLStreamException { + private AggregatedTestResult toAggregatedResult(TestIdentifier testIdentifier) { + if (this.reportData.wasSkipped(testIdentifier)) { + return AggregatedTestResult.skipped(); + } + return AggregatedTestResult.nonSkipped(this.reportData.getResults(testIdentifier)); + } + + private boolean shouldInclude(TestPlan testPlan, TestIdentifier testIdentifier) { + return testIdentifier.isTest() || testPlan.getChildren(testIdentifier).isEmpty(); + } + + private void writeXmlReport(TestIdentifier testIdentifier, Map tests, + Writer out) throws XMLStreamException { XMLOutputFactory factory = XMLOutputFactory.newInstance(); XMLStreamWriter xmlWriter = factory.createXMLStreamWriter(out); @@ -85,8 +111,8 @@ private void writeXmlReport(TestIdentifier testIdentifier, List xmlWriter.close(); } - private void writeTestsuite(TestIdentifier testIdentifier, List tests, XMLStreamWriter writer) - throws XMLStreamException { + private void writeTestsuite(TestIdentifier testIdentifier, Map tests, + XMLStreamWriter writer) throws XMLStreamException { // NumberFormat is not thread-safe. Thus, we instantiate it here and pass it to // writeTestcase instead of using a constant @@ -94,13 +120,13 @@ private void writeTestsuite(TestIdentifier testIdentifier, List writer.writeStartElement("testsuite"); - writeSuiteAttributes(testIdentifier, tests, numberFormat, writer); + writeSuiteAttributes(testIdentifier, tests.values(), numberFormat, writer); newLine(writer); writeSystemProperties(writer); - for (TestIdentifier test : tests) { - writeTestcase(test, numberFormat, writer); + for (Entry entry : tests.entrySet()) { + writeTestcase(entry.getKey(), entry.getValue(), numberFormat, writer); } writeOutputElement("system-out", formatNonStandardAttributesAsString(testIdentifier), writer); @@ -109,22 +135,24 @@ private void writeTestsuite(TestIdentifier testIdentifier, List newLine(writer); } - private void writeSuiteAttributes(TestIdentifier testIdentifier, List tests, + private void writeSuiteAttributes(TestIdentifier testIdentifier, Collection testResults, NumberFormat numberFormat, XMLStreamWriter writer) throws XMLStreamException { writeAttributeSafely(writer, "name", testIdentifier.getDisplayName()); - writeTestCounts(tests, writer); + writeTestCounts(testResults, writer); writeAttributeSafely(writer, "time", getTime(testIdentifier, numberFormat)); writeAttributeSafely(writer, "hostname", getHostname().orElse("")); writeAttributeSafely(writer, "timestamp", ISO_LOCAL_DATE_TIME.format(getCurrentDateTime())); } - private void writeTestCounts(List tests, XMLStreamWriter writer) throws XMLStreamException { - TestCounts testCounts = TestCounts.from(this.reportData, tests); - writeAttributeSafely(writer, "tests", String.valueOf(testCounts.getTotal())); - writeAttributeSafely(writer, "skipped", String.valueOf(testCounts.getSkipped())); - writeAttributeSafely(writer, "failures", String.valueOf(testCounts.getFailures())); - writeAttributeSafely(writer, "errors", String.valueOf(testCounts.getErrors())); + private void writeTestCounts(Collection testResults, XMLStreamWriter writer) + throws XMLStreamException { + Map counts = testResults.stream().map(it -> it.type).collect(groupingBy(identity(), counting())); + long total = counts.values().stream().mapToLong(Long::longValue).sum(); + writeAttributeSafely(writer, "tests", String.valueOf(total)); + writeAttributeSafely(writer, "skipped", counts.getOrDefault(SKIPPED, 0L).toString()); + writeAttributeSafely(writer, "failures", counts.getOrDefault(FAILURE, 0L).toString()); + writeAttributeSafely(writer, "errors", counts.getOrDefault(ERROR, 0L).toString()); } private void writeSystemProperties(XMLStreamWriter writer) throws XMLStreamException { @@ -141,8 +169,8 @@ private void writeSystemProperties(XMLStreamWriter writer) throws XMLStreamExcep newLine(writer); } - private void writeTestcase(TestIdentifier testIdentifier, NumberFormat numberFormat, XMLStreamWriter writer) - throws XMLStreamException { + private void writeTestcase(TestIdentifier testIdentifier, AggregatedTestResult testResult, + NumberFormat numberFormat, XMLStreamWriter writer) throws XMLStreamException { writer.writeStartElement("testcase"); @@ -151,7 +179,7 @@ private void writeTestcase(TestIdentifier testIdentifier, NumberFormat numberFor writeAttributeSafely(writer, "time", getTime(testIdentifier, numberFormat)); newLine(writer); - writeSkippedOrErrorOrFailureElement(testIdentifier, writer); + writeSkippedOrErrorOrFailureElement(testIdentifier, testResult, writer); List systemOutElements = new ArrayList<>(); List systemErrElements = new ArrayList<>(); @@ -172,16 +200,18 @@ private String getClassName(TestIdentifier testIdentifier) { return LegacyReportingUtils.getClassName(this.reportData.getTestPlan(), testIdentifier); } - private void writeSkippedOrErrorOrFailureElement(TestIdentifier testIdentifier, XMLStreamWriter writer) - throws XMLStreamException { + private void writeSkippedOrErrorOrFailureElement(TestIdentifier testIdentifier, AggregatedTestResult testResult, + XMLStreamWriter writer) throws XMLStreamException { - if (this.reportData.wasSkipped(testIdentifier)) { + if (testResult.type == SKIPPED) { writeSkippedElement(this.reportData.getSkipReason(testIdentifier), writer); } else { - Optional result = this.reportData.getResult(testIdentifier); - if (result.isPresent() && result.get().getStatus() == FAILED) { - writeErrorOrFailureElement(result.get(), writer); + Map>> throwablesByType = testResult.getThrowablesByType(); + for (Type type : EnumSet.of(FAILURE, ERROR)) { + for (Optional throwable : throwablesByType.getOrDefault(type, emptyList())) { + writeErrorOrFailureElement(type, throwable.orElse(null), writer); + } } } } @@ -198,17 +228,17 @@ private void writeSkippedElement(String reason, XMLStreamWriter writer) throws X newLine(writer); } - private void writeErrorOrFailureElement(TestExecutionResult result, XMLStreamWriter writer) + private void writeErrorOrFailureElement(Type type, Throwable throwable, XMLStreamWriter writer) throws XMLStreamException { - Optional throwable = result.getThrowable(); - if (throwable.isPresent()) { - writer.writeStartElement(isFailure(result) ? "failure" : "error"); - writeFailureAttributesAndContent(throwable.get(), writer); + String elementName = type == FAILURE ? "failure" : "error"; + if (throwable != null) { + writer.writeStartElement(elementName); + writeFailureAttributesAndContent(throwable, writer); writer.writeEndElement(); } else { - writer.writeEmptyElement("error"); + writer.writeEmptyElement(elementName); } newLine(writer); } @@ -342,54 +372,46 @@ private static boolean isFailure(TestExecutionResult result) { return throwable.isPresent() && throwable.get() instanceof AssertionError; } - private static class TestCounts { - - static TestCounts from(XmlReportData reportData, List tests) { - TestCounts counts = new TestCounts(tests.size()); - for (TestIdentifier test : tests) { - if (reportData.wasSkipped(test)) { - counts.skipped++; - } - else { - Optional result = reportData.getResult(test); - if (result.isPresent() && result.get().getStatus() == FAILED) { - if (isFailure(result.get())) { - counts.failures++; - } - else { - counts.errors++; - } - } - } - } - return counts; - } + static class AggregatedTestResult { - private final long total; - private long skipped; - private long failures; - private long errors; + private static final AggregatedTestResult SKIPPED_RESULT = new AggregatedTestResult(SKIPPED, emptyList()); - TestCounts(long total) { - this.total = total; + public static AggregatedTestResult skipped() { + return SKIPPED_RESULT; } - public long getTotal() { - return total; + public static AggregatedTestResult nonSkipped(List executionResults) { + Type type = executionResults.stream() // + .map(Type::from) // + .max(naturalOrder()) // + .orElse(SUCCESS); + return new AggregatedTestResult(type, executionResults); } - public long getSkipped() { - return skipped; - } + private final Type type; + private final List executionResults; - public long getFailures() { - return failures; + private AggregatedTestResult(Type type, List executionResults) { + this.type = type; + this.executionResults = executionResults; } - public long getErrors() { - return errors; + public Map>> getThrowablesByType() { + return executionResults.stream() // + .collect(groupingBy(Type::from, mapping(TestExecutionResult::getThrowable, toList()))); } + enum Type { + + SUCCESS, SKIPPED, FAILURE, ERROR; + + private static Type from(TestExecutionResult executionResult) { + if (executionResult.getStatus() == FAILED) { + return isFailure(executionResult) ? FAILURE : ERROR; + } + return SUCCESS; + } + } } } diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java index e0c73d2039f2..546cb1331c1f 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java @@ -45,6 +45,9 @@ import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.junit.platform.engine.support.hierarchical.DemoEngineExecutionContext; +import org.junit.platform.engine.support.hierarchical.DemoHierarchicalContainerDescriptor; +import org.junit.platform.engine.support.hierarchical.DemoHierarchicalTestDescriptor; import org.junit.platform.engine.support.hierarchical.DemoHierarchicalTestEngine; import org.junit.platform.fakes.TestDescriptorStub; import org.junit.platform.launcher.TestIdentifier; @@ -58,8 +61,11 @@ */ class LegacyXmlReportGeneratingListenerTests { + @TempDir + Path tempDirectory; + @Test - void writesFileForSingleSucceedingTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleSucceedingTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("succeedingTest", "display<-->Name 😎", () -> { }); @@ -89,7 +95,7 @@ void writesFileForSingleSucceedingTest(@TempDir Path tempDirectory) throws Excep } @Test - void writesFileForSingleFailingTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleFailingTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("failingTest", () -> fail("expected to fail")); @@ -115,7 +121,7 @@ void writesFileForSingleFailingTest(@TempDir Path tempDirectory) throws Exceptio } @Test - void writesFileForSingleErroneousTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleErroneousTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("failingTest", () -> { throw new RuntimeException("error occurred"); @@ -143,7 +149,7 @@ void writesFileForSingleErroneousTest(@TempDir Path tempDirectory) throws Except } @Test - void writesFileForSingleSkippedTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleSkippedTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); var testDescriptor = engine.addTest("skippedTest", () -> fail("never called")); testDescriptor.markSkipped("should be skipped"); @@ -167,7 +173,7 @@ void writesFileForSingleSkippedTest(@TempDir Path tempDirectory) throws Exceptio @SuppressWarnings("ConstantConditions") @Test - void writesFileForSingleAbortedTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleAbortedTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("abortedTest", () -> assumeFalse(true, "deliberately aborted")); @@ -190,7 +196,7 @@ void writesFileForSingleAbortedTest(@TempDir Path tempDirectory) throws Exceptio } @Test - void measuresTimesInSeconds(@TempDir Path tempDirectory) throws Exception { + void measuresTimesInSeconds() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("firstTest", () -> { }); @@ -216,7 +222,7 @@ void measuresTimesInSeconds(@TempDir Path tempDirectory) throws Exception { } @Test - void testWithImmeasurableTimeIsOutputCorrectly(@TempDir Path tempDirectory) throws Exception { + void testWithImmeasurableTimeIsOutputCorrectly() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -229,7 +235,7 @@ void testWithImmeasurableTimeIsOutputCorrectly(@TempDir Path tempDirectory) thro } @Test - void writesFileForSkippedContainer(@TempDir Path tempDirectory) throws Exception { + void writesFileForSkippedContainer() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> fail("never called")); engine.getEngineDescriptor().markSkipped("should be skipped"); @@ -247,7 +253,7 @@ void writesFileForSkippedContainer(@TempDir Path tempDirectory) throws Exception } @Test - void writesFileForFailingContainer(@TempDir Path tempDirectory) throws Exception { + void writesFileForFailingContainer() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> fail("never called")); engine.getEngineDescriptor().setBeforeAllBehavior(() -> fail("failure before all tests")); @@ -269,7 +275,63 @@ void writesFileForFailingContainer(@TempDir Path tempDirectory) throws Exception } @Test - void writesSystemProperties(@TempDir Path tempDirectory) throws Exception { + void writesFileForFailingContainerWithoutTest() throws Exception { + var engine = new DemoHierarchicalTestEngine("dummy"); + engine.addContainer("failingContainer", () -> { + throw new RuntimeException("boom"); + }); + + executeTests(engine, tempDirectory); + + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("errors", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("failingContainer"); + assertThat(testcase.attr("classname")).isEqualTo("dummy"); + + var error = testcase.child("error"); + assertThat(error.attr("message")).isEqualTo("boom"); + assertThat(error.attr("type")).isEqualTo(RuntimeException.class.getName()); + assertThat(error.text()).containsSubsequence("RuntimeException: boom", "\tat"); + } + + @Test + void writesFileForContainerFailingAfterTest() throws Exception { + var engine = new DemoHierarchicalTestEngine("dummy"); + + var container = engine.addChild("failingContainer", + uniqueId -> new DemoHierarchicalContainerDescriptor(uniqueId, "failingContainer", null, null) { + @Override + public void after(DemoEngineExecutionContext context) { + throw new RuntimeException("boom"); + } + }, "child"); + container.addChild( + new DemoHierarchicalTestDescriptor(container.getUniqueId().append("test", "someTest"), "someTest", () -> { + })); + + executeTests(engine, tempDirectory); + + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("errors", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("someTest"); + assertThat(testcase.attr("classname")).isEqualTo("failingContainer"); + + var error = testcase.child("error"); + assertThat(error.attr("message")).isEqualTo("boom"); + assertThat(error.attr("type")).isEqualTo(RuntimeException.class.getName()); + assertThat(error.text()).containsSubsequence("RuntimeException: boom", "\tat"); + } + + @Test + void writesSystemProperties() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -283,7 +345,7 @@ void writesSystemProperties(@TempDir Path tempDirectory) throws Exception { } @Test - void writesHostNameAndTimestamp(@TempDir Path tempDirectory) throws Exception { + void writesHostNameAndTimestamp() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -299,7 +361,7 @@ void writesHostNameAndTimestamp(@TempDir Path tempDirectory) throws Exception { } @Test - void printsExceptionWhenReportsDirCannotBeCreated(@TempDir Path tempDirectory) throws Exception { + void printsExceptionWhenReportsDirCannotBeCreated() throws Exception { var reportsDir = tempDirectory.resolve("dummy.txt"); Files.write(reportsDir, Set.of("content")); @@ -313,7 +375,7 @@ void printsExceptionWhenReportsDirCannotBeCreated(@TempDir Path tempDirectory) t } @Test - void printsExceptionWhenReportCouldNotBeWritten(@TempDir Path tempDirectory) throws Exception { + void printsExceptionWhenReportCouldNotBeWritten() throws Exception { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); var xmlFile = tempDirectory.resolve("TEST-engine.xml"); @@ -329,7 +391,7 @@ void printsExceptionWhenReportCouldNotBeWritten(@TempDir Path tempDirectory) thr } @Test - void writesReportEntriesToSystemOutElement(@TempDir Path tempDirectory) throws Exception { + void writesReportEntriesToSystemOutElement() throws Exception { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java index 7f339554a6ed..0a528f543559 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java @@ -29,19 +29,19 @@ class XmlReportDataTests { @Test - void resultOfTestIdentifierWithoutAnyReportedEventsIsEmpty() { + void resultsOfTestIdentifierWithoutAnyReportedEventsAreEmpty() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).isEmpty(); + assertThat(results).isEmpty(); } @Test - void resultOfTestIdentifierWithoutReportedEventsIsFailureOfAncestor() { + void resultsOfTestIdentifierWithoutReportedEventsContainsOnlyFailureOfAncestor() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -50,13 +50,13 @@ void resultOfTestIdentifierWithoutReportedEventsIsFailureOfAncestor() { var failureOfAncestor = failed(new RuntimeException("failed!")); reportData.markFinished(testPlan.getTestIdentifier("[engine:engine]"), failureOfAncestor); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).contains(failureOfAncestor); + assertThat(results).containsExactly(failureOfAncestor); } @Test - void resultOfTestIdentifierWithoutReportedEventsIsEmptyWhenAncestorWasSuccessful() { + void resultsOfTestIdentifierWithoutReportedEventsContainsOnlySuccessOfAncestor() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -64,8 +64,8 @@ void resultOfTestIdentifierWithoutReportedEventsIsEmptyWhenAncestorWasSuccessful var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); reportData.markFinished(testPlan.getTestIdentifier("[engine:engine]"), successful()); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).isEmpty(); + assertThat(results).containsExactly(successful()); } }