From 47aebbb6d7faad3db837fe5fe93d8ffb11bf37c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tibor=20Diga=C5=88a?= Date: Fri, 28 Jan 2022 01:04:12 +0100 Subject: [PATCH] [SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an assumption --- .../surefire/report/StatelessXmlReporter.java | 54 +++++++--- .../report/StatelessXmlReporterTest.java | 100 +++++++++++++++--- 2 files changed, 127 insertions(+), 27 deletions(-) diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java index 984bd94178..45f1c5003c 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.Deque; import java.util.LinkedHashMap; @@ -44,6 +45,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType; import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars; +import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED; import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS; import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank; @@ -85,6 +87,10 @@ public class StatelessXmlReporter implements StatelessReportEventListener { + private static final String XML_INDENT = " "; + + private static final String XML_NL = "\n"; + private final File reportsDirectory; private final String reportNameSuffix; @@ -139,8 +145,7 @@ public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStat try ( OutputStream outputStream = getOutputStream( testSetReportEntry ); OutputStreamWriter fw = getWriter( outputStream ) ) { - XMLWriter ppw = new PrettyPrintXMLWriter( fw ); - ppw.setEncoding( UTF_8.name() ); + XMLWriter ppw = new PrettyPrintXMLWriter( new PrintWriter( fw ), XML_INDENT, XML_NL, UTF_8.name(), null ); createTestSuiteElement( ppw, testSetReportEntry, testSetStats ); // TestSuite @@ -264,6 +269,14 @@ private void serializeTestClassWithRerun( OutputStream outputStream, OutputStrea singleRunEntry.getReportEntryType().getXmlTag(), false ); createOutErrElements( fw, ppw, singleRunEntry, outputStream ); } + else if ( singleRunEntry.getReportEntryType() == SKIPPED ) + { + // The version 3.1.0 should produce a new XSD schema with version 3.1.0, see SUREFIRE-1986, + // and the XSD schema should add a new element "rerunSkipped" + // then ReportEntryType should update the enum to SKIPPED( "skipped", "", "rerunSkipped" ). + // The teams should be notified - Jenkins reports. + addCommentElementTestCase( "a skipped test execution in re-run phase", fw, ppw, outputStream ); + } else { getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream, @@ -347,6 +360,7 @@ private OutputStream getOutputStream( WrappedReportEntry testSetReportEntry ) { File reportFile = getReportFile( testSetReportEntry ); File reportDir = reportFile.getParentFile(); + //noinspection ResultOfMethodCallIgnored reportFile.delete(); //noinspection ResultOfMethodCallIgnored reportDir.mkdirs(); @@ -572,6 +586,24 @@ private static void extraEscapeElementValue( String message, OutputStreamWriter } } + // todo: SUREFIRE-1986 + private static void addCommentElementTestCase( String comment, OutputStreamWriter outputStreamWriter, + XMLWriter xmlWriter, OutputStream fw ) + throws IOException + { + xmlWriter.writeText( "" ); // Cheat sax to emit element + outputStreamWriter.flush(); + fw.write( XML_NL.getBytes( UTF_8 ) ); + fw.write( XML_INDENT.getBytes( UTF_8 ) ); + fw.write( XML_INDENT.getBytes( UTF_8 ) ); + fw.write( ByteConstantsHolder.COMMENT_START ); + fw.write( comment.getBytes( UTF_8 ) ); + fw.write( ByteConstantsHolder.COMMENT_END ); + fw.write( XML_NL.getBytes( UTF_8 ) ); + fw.write( XML_INDENT.getBytes( UTF_8 ) ); + fw.flush(); + } + private static final class EncodingOutputStream extends FilterOutputStream { @@ -679,20 +711,16 @@ private static String escapeXml( String text, boolean attribute ) private static final class ByteConstantsHolder { - private static final byte[] CDATA_START_BYTES; + private static final byte[] CDATA_START_BYTES = "".getBytes( UTF_8 ); - private static final byte[] CDATA_ESCAPE_STRING_BYTES; + private static final byte[] CDATA_ESCAPE_STRING_BYTES = "]]>".getBytes( UTF_8 ); - private static final byte[] AMP_BYTES; + private static final byte[] AMP_BYTES = "&#".getBytes( UTF_8 ); - static - { - CDATA_START_BYTES = "".getBytes( UTF_8 ); - CDATA_ESCAPE_STRING_BYTES = "]]>".getBytes( UTF_8 ); - AMP_BYTES = "&#".getBytes( UTF_8 ); - } + private static final byte[] COMMENT_START = " ".getBytes( UTF_8 ); } } diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java index 7bdb23540e..4358a39ae6 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java @@ -24,7 +24,6 @@ import org.apache.maven.surefire.api.report.ReportEntry; import org.apache.maven.surefire.api.report.SimpleReportEntry; import org.apache.maven.surefire.api.report.StackTraceWriter; -import org.apache.maven.surefire.shared.utils.StringUtils; import org.apache.maven.surefire.shared.utils.xml.Xpp3Dom; import org.apache.maven.surefire.shared.utils.xml.Xpp3DomBuilder; @@ -43,7 +42,13 @@ import java.util.concurrent.atomic.AtomicInteger; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.readAllLines; +import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR; +import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE; +import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED; +import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS; import static org.apache.maven.surefire.api.util.internal.ObjectUtils.systemProps; +import static org.apache.maven.surefire.shared.utils.StringUtils.isEmpty; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -58,7 +63,7 @@ /** * */ -@SuppressWarnings( "ResultOfMethodCallIgnored" ) +@SuppressWarnings( { "ResultOfMethodCallIgnored", "checkstyle:magicnumber" } ) public class StatelessXmlReporterTest extends TestCase { @@ -107,7 +112,7 @@ public void testFileNameWithoutSuffix() reporter.cleanTestHistoryMap(); ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, getClass().getName(), null, 12 ); - WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, + WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, SUCCESS, 12, null, null, systemProps() ); stats.testSucceeded( testSetReportEntry ); reporter.testSetCompleted( testSetReportEntry, stats ); @@ -123,7 +128,7 @@ public void testAllFieldsSerialized() { ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 ); WrappedReportEntry testSetReportEntry = - new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, 12, null, null, systemProps() ); + new WrappedReportEntry( reportEntry, SUCCESS, 12, null, null, systemProps() ); expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" ); stats.testSucceeded( testSetReportEntry ); @@ -149,7 +154,7 @@ public void testAllFieldsSerialized() stdErr.write( stdErrPrefix + "?&-&£\u0020\u0000\u001F", false ); WrappedReportEntry t2 = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null, - stackTraceWriter, 13 ), ReportEntryType.ERROR, 13, stdOut, stdErr ); + stackTraceWriter, 13 ), ERROR, 13, stdOut, stdErr ); stats.testSucceeded( t2 ); StatelessXmlReporter reporter = new StatelessXmlReporter( reportDir, null, false, 0, @@ -163,8 +168,8 @@ public void testAllFieldsSerialized() Xpp3Dom properties = testSuite.getChild( "properties" ); assertEquals( System.getProperties().size(), properties.getChildCount() ); Xpp3Dom child = properties.getChild( 1 ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) ); + assertFalse( isEmpty( child.getAttribute( "value" ) ) ); + assertFalse( isEmpty( child.getAttribute( "name" ) ) ); Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); Xpp3Dom tca = testcase[0]; @@ -191,7 +196,7 @@ public void testOutputRerunFlakyFailure() { WrappedReportEntry testSetReportEntry = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 ), - ReportEntryType.SUCCESS, 12, null, null, systemProps() ); + SUCCESS, 12, null, null, systemProps() ); expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" ); stats.testSucceeded( testSetReportEntry ); @@ -207,22 +212,22 @@ public void testOutputRerunFlakyFailure() WrappedReportEntry testTwoFirstError = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null, - stackTraceWriterOne, 5 ), ReportEntryType.ERROR, 5, createStdOutput( firstRunOut ), + stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ), createStdOutput( firstRunErr ) ); WrappedReportEntry testTwoSecondError = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null, - stackTraceWriterTwo, 13 ), ReportEntryType.ERROR, 13, createStdOutput( secondRunOut ), + stackTraceWriterTwo, 13 ), ERROR, 13, createStdOutput( secondRunOut ), createStdOutput( secondRunErr ) ); WrappedReportEntry testThreeFirstRun = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null, - stackTraceWriterOne, 13 ), ReportEntryType.FAILURE, 13, createStdOutput( firstRunOut ), + stackTraceWriterOne, 13 ), FAILURE, 13, createStdOutput( firstRunOut ), createStdOutput( firstRunErr ) ); WrappedReportEntry testThreeSecondRun = new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null, - stackTraceWriterTwo, 2 ), ReportEntryType.SUCCESS, 2, createStdOutput( secondRunOut ), + stackTraceWriterTwo, 2 ), SUCCESS, 2, createStdOutput( secondRunOut ), createStdOutput( secondRunErr ) ); stats.testSucceeded( testTwoFirstError ); @@ -245,8 +250,8 @@ public void testOutputRerunFlakyFailure() Xpp3Dom properties = testSuite.getChild( "properties" ); assertEquals( System.getProperties().size(), properties.getChildCount() ); Xpp3Dom child = properties.getChild( 1 ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) ); - assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) ); + assertFalse( isEmpty( child.getAttribute( "value" ) ) ); + assertFalse( isEmpty( child.getAttribute( "name" ) ) ); Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); Xpp3Dom testCaseOne = testcase[0]; @@ -290,6 +295,73 @@ public void testOutputRerunFlakyFailure() assertNull( testCaseThree.getChild( "system-err" ) ); } + public void testOutputRerunFlakyAssumption() + throws IOException + { + expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" ); + + StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed", + "fail at foo" ); + + StackTraceWriter stackTraceWriterTwo = + new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" ); + + String firstRunOut = "first run out"; + String firstRunErr = "first run err"; + String secondRunOut = "second run out"; + String secondRunErr = "second run err"; + + WrappedReportEntry testTwoFirstError = + new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null, + stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ), + createStdOutput( firstRunErr ) ); + + stats.testSucceeded( testTwoFirstError ); + + WrappedReportEntry testTwoSecondError = + new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null, + stackTraceWriterTwo, 13 ), SKIPPED, 13, createStdOutput( secondRunOut ), + createStdOutput( secondRunErr ) ); + + rerunStats.testSucceeded( testTwoSecondError ); + + StatelessXmlReporter reporter = + new StatelessXmlReporter( reportDir, null, false, 1, + new HashMap<>(), XSD, "3.0", false, false, false, false ); + + WrappedReportEntry testSetReportEntry = + new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, null, null, + stackTraceWriterOne, 5 ), ERROR, 20, createStdOutput( firstRunOut ), + createStdOutput( firstRunErr ) ); + + reporter.testSetCompleted( testSetReportEntry, stats ); + reporter.testSetCompleted( testSetReportEntry, rerunStats ); + + FileInputStream fileInputStream = new FileInputStream( expectedReportFile ); + + Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, UTF_8 ) ); + assertEquals( "testsuite", testSuite.getName() ); + assertEquals( "0.02", testSuite.getAttribute( "time" ) ); + + Xpp3Dom[] testcase = testSuite.getChildren( "testcase" ); + assertEquals( 1, testcase.length ); + Xpp3Dom testCaseOne = testcase[0]; + assertEquals( getClass().getName(), testCaseOne.getAttribute( "classname" ) ); + assertEquals( TEST_TWO, testCaseOne.getAttribute( "name" ) ); + assertEquals( "0.005", testCaseOne.getAttribute( "time" ) ); + + Xpp3Dom[] testCaseElements = testCaseOne.getChildren(); + assertEquals( 3, testCaseElements.length ); + assertEquals( "error", testCaseElements[0].getName() ); + assertEquals( "system-out", testCaseElements[1].getName() ); + assertEquals( "system-err", testCaseElements[2].getName() ); + long linesWithComments = readAllLines( expectedReportFile.toPath(), UTF_8 ) + .stream() + .filter( line -> line.contains( "" ) ) + .count(); + assertEquals( 1, linesWithComments ); + } + public void testNoWritesOnDeferredFile() throws Exception { Utf8RecodingDeferredFileOutputStream out = new Utf8RecodingDeferredFileOutputStream( "test" );