Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an assumption #450

Merged
merged 1 commit into from Feb 2, 2022
Merged

[SUREFIRE-1556] Test XML file is not valid when rerun "fails" with an assumption #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
54 changes: 41 additions & 13 deletions ...re-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
View file
Open in desktop
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Deque;
import java.util.LinkedHashMap;
Expand All @@ -44,6 +45,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType;
import static org.apache.maven.plugin.surefire.report.FileReporterUtils.stripIllegalFilenameChars;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank;

Expand Down Expand Up @@ -85,6 +87,10 @@
public class StatelessXmlReporter
implements StatelessReportEventListener<WrappedReportEntry, TestSetStats>
{
private static final String XML_INDENT = " ";

private static final String XML_NL = "\n";

private final File reportsDirectory;

private final String reportNameSuffix;
Expand Down Expand Up @@ -139,8 +145,7 @@ public void testSetCompleted( WrappedReportEntry testSetReportEntry, TestSetStat
try ( OutputStream outputStream = getOutputStream( testSetReportEntry );
OutputStreamWriter fw = getWriter( outputStream ) )
{
XMLWriter ppw = new PrettyPrintXMLWriter( fw );
ppw.setEncoding( UTF_8.name() );
XMLWriter ppw = new PrettyPrintXMLWriter( new PrintWriter( fw ), XML_INDENT, XML_NL, UTF_8.name(), null );

createTestSuiteElement( ppw, testSetReportEntry, testSetStats ); // TestSuite

Expand Down Expand Up @@ -264,6 +269,14 @@ private void serializeTestClassWithRerun( OutputStream outputStream, OutputStrea
singleRunEntry.getReportEntryType().getXmlTag(), false );
createOutErrElements( fw, ppw, singleRunEntry, outputStream );
}
else if ( singleRunEntry.getReportEntryType() == SKIPPED )
{
// The version 3.1.0 should produce a new XSD schema with version 3.1.0, see SUREFIRE-1986,
// and the XSD schema should add a new element "rerunSkipped"
// then ReportEntryType should update the enum to SKIPPED( "skipped", "", "rerunSkipped" ).
// The teams should be notified - Jenkins reports.
addCommentElementTestCase( "a skipped test execution in re-run phase", fw, ppw, outputStream );
}
else
{
getTestProblems( fw, ppw, singleRunEntry, trimStackTrace, outputStream,
Expand Down Expand Up @@ -347,6 +360,7 @@ private OutputStream getOutputStream( WrappedReportEntry testSetReportEntry )
{
File reportFile = getReportFile( testSetReportEntry );
File reportDir = reportFile.getParentFile();
//noinspection ResultOfMethodCallIgnored
reportFile.delete();
//noinspection ResultOfMethodCallIgnored
reportDir.mkdirs();
Expand Down Expand Up @@ -572,6 +586,24 @@ private static void extraEscapeElementValue( String message, OutputStreamWriter
}
}

// todo: SUREFIRE-1986
private static void addCommentElementTestCase( String comment, OutputStreamWriter outputStreamWriter,
XMLWriter xmlWriter, OutputStream fw )
throws IOException
{
xmlWriter.writeText( "" ); // Cheat sax to emit element
outputStreamWriter.flush();
fw.write( XML_NL.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.write( ByteConstantsHolder.COMMENT_START );
fw.write( comment.getBytes( UTF_8 ) );
fw.write( ByteConstantsHolder.COMMENT_END );
fw.write( XML_NL.getBytes( UTF_8 ) );
fw.write( XML_INDENT.getBytes( UTF_8 ) );
fw.flush();
}

private static final class EncodingOutputStream
extends FilterOutputStream
{
Expand Down Expand Up @@ -679,20 +711,16 @@ private static String escapeXml( String text, boolean attribute )

private static final class ByteConstantsHolder
{
private static final byte[] CDATA_START_BYTES;
private static final byte[] CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );

private static final byte[] CDATA_END_BYTES;
private static final byte[] CDATA_END_BYTES = "]]>".getBytes( UTF_8 );

private static final byte[] CDATA_ESCAPE_STRING_BYTES;
private static final byte[] CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );

private static final byte[] AMP_BYTES;
private static final byte[] AMP_BYTES = "&amp#".getBytes( UTF_8 );

static
{
CDATA_START_BYTES = "<![CDATA[".getBytes( UTF_8 );
CDATA_END_BYTES = "]]>".getBytes( UTF_8 );
CDATA_ESCAPE_STRING_BYTES = "]]><![CDATA[>".getBytes( UTF_8 );
AMP_BYTES = "&amp#".getBytes( UTF_8 );
}
private static final byte[] COMMENT_START = "<!-- ".getBytes( UTF_8 );

private static final byte[] COMMENT_END = " --> ".getBytes( UTF_8 );
}
}
100 changes: 86 additions & 14 deletions ...ommon/src/test/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporterTest.java
View file
Open in desktop
Expand Up @@ -24,7 +24,6 @@
import org.apache.maven.surefire.api.report.ReportEntry;
import org.apache.maven.surefire.api.report.SimpleReportEntry;
import org.apache.maven.surefire.api.report.StackTraceWriter;
import org.apache.maven.surefire.shared.utils.StringUtils;
import org.apache.maven.surefire.shared.utils.xml.Xpp3Dom;
import org.apache.maven.surefire.shared.utils.xml.Xpp3DomBuilder;

Expand All @@ -43,7 +42,13 @@
import java.util.concurrent.atomic.AtomicInteger;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.Files.readAllLines;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.ERROR;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.FAILURE;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SKIPPED;
import static org.apache.maven.plugin.surefire.report.ReportEntryType.SUCCESS;
import static org.apache.maven.surefire.api.util.internal.ObjectUtils.systemProps;
import static org.apache.maven.surefire.shared.utils.StringUtils.isEmpty;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
Expand All @@ -58,7 +63,7 @@
/**
*
*/
@SuppressWarnings( "ResultOfMethodCallIgnored" )
@SuppressWarnings( { "ResultOfMethodCallIgnored", "checkstyle:magicnumber" } )
public class StatelessXmlReporterTest
extends TestCase
{
Expand Down Expand Up @@ -107,7 +112,7 @@ public void testFileNameWithoutSuffix()
reporter.cleanTestHistoryMap();

ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, getClass().getName(), null, 12 );
WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS,
WrappedReportEntry testSetReportEntry = new WrappedReportEntry( reportEntry, SUCCESS,
12, null, null, systemProps() );
stats.testSucceeded( testSetReportEntry );
reporter.testSetCompleted( testSetReportEntry, stats );
Expand All @@ -123,7 +128,7 @@ public void testAllFieldsSerialized()
{
ReportEntry reportEntry = new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 );
WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( reportEntry, ReportEntryType.SUCCESS, 12, null, null, systemProps() );
new WrappedReportEntry( reportEntry, SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

stats.testSucceeded( testSetReportEntry );
Expand All @@ -149,7 +154,7 @@ public void testAllFieldsSerialized()
stdErr.write( stdErrPrefix + "?&-&amp;&#163;\u0020\u0000\u001F", false );
WrappedReportEntry t2 =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriter, 13 ), ReportEntryType.ERROR, 13, stdOut, stdErr );
stackTraceWriter, 13 ), ERROR, 13, stdOut, stdErr );

stats.testSucceeded( t2 );
StatelessXmlReporter reporter = new StatelessXmlReporter( reportDir, null, false, 0,
Expand All @@ -163,8 +168,8 @@ public void testAllFieldsSerialized()
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
assertFalse( isEmpty( child.getAttribute( "value" ) ) );
assertFalse( isEmpty( child.getAttribute( "name" ) ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom tca = testcase[0];
Expand All @@ -191,7 +196,7 @@ public void testOutputRerunFlakyFailure()
{
WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_ONE, null, 12 ),
ReportEntryType.SUCCESS, 12, null, null, systemProps() );
SUCCESS, 12, null, null, systemProps() );
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

stats.testSucceeded( testSetReportEntry );
Expand All @@ -207,22 +212,22 @@ public void testOutputRerunFlakyFailure()

WrappedReportEntry testTwoFirstError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterOne, 5 ), ReportEntryType.ERROR, 5, createStdOutput( firstRunOut ),
stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

WrappedReportEntry testTwoSecondError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterTwo, 13 ), ReportEntryType.ERROR, 13, createStdOutput( secondRunOut ),
stackTraceWriterTwo, 13 ), ERROR, 13, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

WrappedReportEntry testThreeFirstRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
stackTraceWriterOne, 13 ), ReportEntryType.FAILURE, 13, createStdOutput( firstRunOut ),
stackTraceWriterOne, 13 ), FAILURE, 13, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

WrappedReportEntry testThreeSecondRun =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_THREE, null,
stackTraceWriterTwo, 2 ), ReportEntryType.SUCCESS, 2, createStdOutput( secondRunOut ),
stackTraceWriterTwo, 2 ), SUCCESS, 2, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

stats.testSucceeded( testTwoFirstError );
Expand All @@ -245,8 +250,8 @@ public void testOutputRerunFlakyFailure()
Xpp3Dom properties = testSuite.getChild( "properties" );
assertEquals( System.getProperties().size(), properties.getChildCount() );
Xpp3Dom child = properties.getChild( 1 );
assertFalse( StringUtils.isEmpty( child.getAttribute( "value" ) ) );
assertFalse( StringUtils.isEmpty( child.getAttribute( "name" ) ) );
assertFalse( isEmpty( child.getAttribute( "value" ) ) );
assertFalse( isEmpty( child.getAttribute( "name" ) ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
Xpp3Dom testCaseOne = testcase[0];
Expand Down Expand Up @@ -290,6 +295,73 @@ public void testOutputRerunFlakyFailure()
assertNull( testCaseThree.getChild( "system-err" ) );
}

public void testOutputRerunFlakyAssumption()
throws IOException
{
expectedReportFile = new File( reportDir, "TEST-" + getClass().getName() + ".xml" );

StackTraceWriter stackTraceWriterOne = new DeserializedStacktraceWriter( "A fud msg", "trimmed",
"fail at foo" );

StackTraceWriter stackTraceWriterTwo =
new DeserializedStacktraceWriter( "A fud msg two", "trimmed two", "fail at foo two" );

String firstRunOut = "first run out";
String firstRunErr = "first run err";
String secondRunOut = "second run out";
String secondRunErr = "second run err";

WrappedReportEntry testTwoFirstError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterOne, 5 ), ERROR, 5, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

stats.testSucceeded( testTwoFirstError );

WrappedReportEntry testTwoSecondError =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, TEST_TWO, null,
stackTraceWriterTwo, 13 ), SKIPPED, 13, createStdOutput( secondRunOut ),
createStdOutput( secondRunErr ) );

rerunStats.testSucceeded( testTwoSecondError );

StatelessXmlReporter reporter =
new StatelessXmlReporter( reportDir, null, false, 1,
new HashMap<>(), XSD, "3.0", false, false, false, false );

WrappedReportEntry testSetReportEntry =
new WrappedReportEntry( new SimpleReportEntry( getClass().getName(), null, null, null,
stackTraceWriterOne, 5 ), ERROR, 20, createStdOutput( firstRunOut ),
createStdOutput( firstRunErr ) );

reporter.testSetCompleted( testSetReportEntry, stats );
reporter.testSetCompleted( testSetReportEntry, rerunStats );

FileInputStream fileInputStream = new FileInputStream( expectedReportFile );

Xpp3Dom testSuite = Xpp3DomBuilder.build( new InputStreamReader( fileInputStream, UTF_8 ) );
assertEquals( "testsuite", testSuite.getName() );
assertEquals( "0.02", testSuite.getAttribute( "time" ) );

Xpp3Dom[] testcase = testSuite.getChildren( "testcase" );
assertEquals( 1, testcase.length );
Xpp3Dom testCaseOne = testcase[0];
assertEquals( getClass().getName(), testCaseOne.getAttribute( "classname" ) );
assertEquals( TEST_TWO, testCaseOne.getAttribute( "name" ) );
assertEquals( "0.005", testCaseOne.getAttribute( "time" ) );

Xpp3Dom[] testCaseElements = testCaseOne.getChildren();
assertEquals( 3, testCaseElements.length );
assertEquals( "error", testCaseElements[0].getName() );
assertEquals( "system-out", testCaseElements[1].getName() );
assertEquals( "system-err", testCaseElements[2].getName() );
long linesWithComments = readAllLines( expectedReportFile.toPath(), UTF_8 )
.stream()
.filter( line -> line.contains( "<!-- a skipped test execution in re-run phase -->" ) )
.count();
assertEquals( 1, linesWithComments );
}

public void testNoWritesOnDeferredFile() throws Exception
{
Utf8RecodingDeferredFileOutputStream out = new Utf8RecodingDeferredFileOutputStream( "test" );
Expand Down