Skip to content

Commit

Permalink
[SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an…
Browse files Browse the repository at this point in the history 
… assumption
  • Loading branch information
@Tibor17
Tibor17 committed Feb 2, 2022
1 parent 33ccb25 commit 47aebbb
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 27 deletions.
54 changes: 41 additions & 13 deletions ...re-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
View file
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Deque;
import java.util.LinkedHashMap;
Expand All @@ -44,6 +45,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank;

Expand Down Expand Up @@ -85,6 +87,10 @@
public class StatelessXmlReporter
implements StatelessReportEventListener<WrappedReportEntry, TestSetStats>
{
private static final String XML_INDENT = " ";

private static final String XML_NL = "\n";

private final File reportsDirectory;

private final String reportNameSuffix;
Expand Down Expand Up @@ -139,8 +145,7 @@ public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStat
try ( OutputStream outputStream = getOutputStream( testSetReportEntry );
OutputStreamWriter fw = getWriter( outputStream ) )
{
XMLWriter ppw = new PrettyPrintXMLWriter( fw );
ppw.setEncoding( UTF_8.name() );
XMLWriter ppw = new PrettyPrintXMLWriter( new PrintWriter( fw ), XML_INDENT, XML_NL, UTF_8.name(), null );

createTestSuiteElement( ppw, testSetReportEntry, testSetStats ); // TestSuite

Expand Down Expand Up @@ -264,6 +269,14 @@ private void serializeTestClassWithRerun( OutputStream outputStream, OutputStrea
singleRunEntry.getReportEntryType().getXmlTag(), false );
createOutErrElements( fw, ppw, singleRunEntry, outputStream );
}
else if ( singleRunEntry.getReportEntryType() == SKIPPED )
{
// The version 3.1.0 should produce a new XSD schema with version 3.1.0, see SUREFIRE-1986,
// and the XSD schema should add a new element "rerunSkipped"
// then ReportEntryType should update the enum to SKIPPED( "skipped", "", "rerunSkipped" ).
// The teams should be notified - Jenkins reports.
addCommentElementTestCase( "a skipped test execution in re-run phase", fw, ppw, outputStream );
}
else
{
getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
Expand Down Expand Up @@ -347,6 +360,7 @@ private OutputStream getOutputStream( WrappedReportEntry testSetReportEntry )
{
File reportFile = getReportFile( testSetReportEntry );
File reportDir = reportFile.getParentFile();
//noinspection ResultOfMethodCallIgnored
reportFile.delete();
//noinspection ResultOfMethodCallIgnored
reportDir.mkdirs();
Expand Down Expand Up @@ -572,6 +586,24 @@ private static void extraEscapeElementValue( String message, OutputStreamWriter
}
}

// todo: SUREFIRE-1986
private static void addCommentElementTestCase( String comment, OutputStreamWriter outputStreamWriter,
XMLWriter xmlWriter, OutputStream fw )
throws IOException
{
xmlWriter.writeText( "" ); // Cheat sax to emit element
outputStreamWriter.flush();
fw.write( XML_NL.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.write( ByteConstantsHolder.COMMENT_START );
fw.write( comment.getBytes( UTF_8 ) );
fw.write( ByteConstantsHolder.COMMENT_END );
fw.write( XML_NL.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.flush();
}

private static final class EncodingOutputStream
extends FilterOutputStream
{
Expand Down Expand Up @@ -679,20 +711,16 @@ private static String escapeXml( String text, boolean attribute )

private static final class ByteConstantsHolder
{
private static final byte[] CDATA_START_BYTES;
private static final byte[] CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );

private static final byte[] CDATA_END_BYTES;
private static final byte[] CDATA_END_BYTES = "]]>".getBytes( UTF_8 );

private static final byte[] CDATA_ESCAPE_STRING_BYTES;
private static final byte[] CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );

private static final byte[] AMP_BYTES;
private static final byte[] AMP_BYTES = "&amp#".getBytes( UTF_8 );

static
{
CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );
CDATA_END_BYTES = "]]>".getBytes( UTF_8 );
CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );
AMP_BYTES = "&amp#".getBytes( UTF_8 );
}
private static final byte[] COMMENT_START = "<!-- ".getBytes( UTF_8 );

private static final byte[] COMMENT_END = " --> ".getBytes( UTF_8 );
}
}
100 changes: 86 additions & 14 deletions ...ommon/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
View file
Expand Up @@ -24,7 +24,6 @@
import org.apache.maven.surefire.api.report.ReportEntry;
import org.apache.maven.surefire.api.report.SimpleReportEntry;
import org.apache.maven.surefire.api.report.StackTraceWriter;
import org.apache.maven.surefire.shared.utils.StringUtils;
import org.apache.maven.surefire.shared.utils.xml.Xpp3Dom;
import org.apache.maven.surefire.shared.utils.xml.Xpp3DomBuilder;

Expand All @@ -43,7 +42,13 @@
import java.util.concurrent.atomic.AtomicInteger;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.Files.readAllLines;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.api.util.internal.ObjectUtils.systemProps;
import static org.apache.maven.surefire.shared.utils.StringUtils.isEmpty;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
Expand All @@ -58,7 +63,7 @@
/**
*
*/
@SuppressWarnings( "ResultOfMethodCallIgnored" )
@SuppressWarnings( { "ResultOfMethodCallIgnored", "checkstyle:magicnumber" } )
public class StatelessXmlReporterTest
extends TestCase
{
Expand Down Expand Up @@ -107,7 +112,7 @@ public void testFileNameWithoutSuffix()
reporter.cleanTestHistoryMap();

ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, getClass().getName(), null, 12 );
WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS,
WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, SUCCESS,
12, null, null, systemProps() );
stats.testSucceeded( testSetReportEntry );
reporter.testSetCompleted( testSetReportEntry, stats );
Expand All @@ -123,7 +128,7 @@ public void testAllFieldsSerialized()
{
ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 );
WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, 12, null, null, systemProps() );
new WrappedReportEntry( reportEntry, SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

stats.testSucceeded( testSetReportEntry );
Expand All @@ -149,7 +154,7 @@ public void testAllFieldsSerialized()
stdErr.write( stdErrPrefix + "?&-&amp;&#163;\u0020\u0000\u001F", false );
WrappedReportEntry t2 =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriter, 13 ), ReportEntryType.ERROR, 13, stdOut, stdErr );
stackTraceWriter, 13 ), ERROR, 13, stdOut, stdErr );

stats.testSucceeded( t2 );
StatelessXmlReporter reporter = new StatelessXmlReporter( reportDir, null, false, 0,
Expand All @@ -163,8 +168,8 @@ public void testAllFieldsSerialized()
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
assertFalse( isEmpty( child.getAttribute( "value" ) ) );
assertFalse( isEmpty( child.getAttribute( "name" ) ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom tca = testcase[0];
Expand All @@ -191,7 +196,7 @@ public void testOutputRerunFlakyFailure()
{
WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 ),
ReportEntryType.SUCCESS, 12, null, null, systemProps() );
SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

stats.testSucceeded( testSetReportEntry );
Expand All @@ -207,22 +212,22 @@ public void testOutputRerunFlakyFailure()

WrappedReportEntry testTwoFirstError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterOne, 5 ), ReportEntryType.ERROR, 5, createStdOutput( firstRunOut ),
stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

WrappedReportEntry testTwoSecondError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterTwo, 13 ), ReportEntryType.ERROR, 13, createStdOutput( secondRunOut ),
stackTraceWriterTwo, 13 ), ERROR, 13, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

WrappedReportEntry testThreeFirstRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
stackTraceWriterOne, 13 ), ReportEntryType.FAILURE, 13, createStdOutput( firstRunOut ),
stackTraceWriterOne, 13 ), FAILURE, 13, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

WrappedReportEntry testThreeSecondRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
stackTraceWriterTwo, 2 ), ReportEntryType.SUCCESS, 2, createStdOutput( secondRunOut ),
stackTraceWriterTwo, 2 ), SUCCESS, 2, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

stats.testSucceeded( testTwoFirstError );
Expand All @@ -245,8 +250,8 @@ public void testOutputRerunFlakyFailure()
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
assertFalse( isEmpty( child.getAttribute( "value" ) ) );
assertFalse( isEmpty( child.getAttribute( "name" ) ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom testCaseOne = testcase[0];
Expand Down Expand Up @@ -290,6 +295,73 @@ public void testOutputRerunFlakyFailure()
assertNull( testCaseThree.getChild( "system-err" ) );
}

public void testOutputRerunFlakyAssumption()
throws IOException
{
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed",
"fail at foo" );

StackTraceWriter stackTraceWriterTwo =
new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" );

String firstRunOut = "first run out";
String firstRunErr = "first run err";
String secondRunOut = "second run out";
String secondRunErr = "second run err";

WrappedReportEntry testTwoFirstError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

stats.testSucceeded( testTwoFirstError );

WrappedReportEntry testTwoSecondError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterTwo, 13 ), SKIPPED, 13, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

rerunStats.testSucceeded( testTwoSecondError );

StatelessXmlReporter reporter =
new StatelessXmlReporter( reportDir, null, false, 1,
new HashMap<>(), XSD, "3.0", false, false, false, false );

WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, null, null,
stackTraceWriterOne, 5 ), ERROR, 20, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

reporter.testSetCompleted( testSetReportEntry, stats );
reporter.testSetCompleted( testSetReportEntry, rerunStats );

FileInputStream fileInputStream = new FileInputStream( expectedReportFile );

Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, UTF_8 ) );
assertEquals( "testsuite", testSuite.getName() );
assertEquals( "0.02", testSuite.getAttribute( "time" ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
assertEquals( 1, testcase.length );
Xpp3Dom testCaseOne = testcase[0];
assertEquals( getClass().getName(), testCaseOne.getAttribute( "classname" ) );
assertEquals( TEST_TWO, testCaseOne.getAttribute( "name" ) );
assertEquals( "0.005", testCaseOne.getAttribute( "time" ) );

Xpp3Dom[] testCaseElements = testCaseOne.getChildren();
assertEquals( 3, testCaseElements.length );
assertEquals( "error", testCaseElements[0].getName() );
assertEquals( "system-out", testCaseElements[1].getName() );
assertEquals( "system-err", testCaseElements[2].getName() );
long linesWithComments = readAllLines( expectedReportFile.toPath(), UTF_8 )
.stream()
.filter( line -> line.contains( "<!-- a skipped test execution in re-run phase -->" ) )
.count();
assertEquals( 1, linesWithComments );
}

public void testNoWritesOnDeferredFile() throws Exception
{
Utf8RecodingDeferredFileOutputStream out = new Utf8RecodingDeferredFileOutputStream( "test" );
Expand Down

0 comments on commit 47aebbb

Please sign in to comment.