diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgMethodParallelOrderingIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgMethodParallelOrderingIT.java new file mode 100644 index 0000000000..14e7fb9b86 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/CheckTestNgMethodParallelOrderingIT.java @@ -0,0 +1,41 @@ +package org.apache.maven.surefire.its; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * Test TestNG setup and teardown ordering with parallelism + * + * @author findepi + */ +public class CheckTestNgMethodParallelOrderingIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testNgParallelOrdering() + { + unpack( "surefire-1967-testng-method-parallel-ordering" ) + .sysProp( "testNgVersion", "7.3.0" ) + .executeTest() + .verifyErrorFree( 12 ); + } +} diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/pom.xml b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/pom.xml new file mode 100644 index 0000000000..2aa3f0881f --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/pom.xml @@ -0,0 +1,82 @@ + + + + + 4.0.0 + + + org.apache.maven.surefire + it-parent + 1.0 + ../pom.xml + + + org.apache.maven.plugins.surefire + surefire-1967-testng-method-parallel-ordering + 1.0-SNAPSHOT + TestNG parallel ordering + + + + testng-old + + testNgClassifier + + + + org.testng + testng + ${testNgVersion} + ${testNgClassifier} + + + + + testng-new + + !testNgClassifier + + + + org.testng + testng + ${testNgVersion} + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + 2 + methods + + + + + + diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/Base.java b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/Base.java new file mode 100644 index 0000000000..147852d338 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/Base.java @@ -0,0 +1,59 @@ +package testng.parallelOrdering; + +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +public abstract class Base +{ + private static final AtomicInteger resources = new AtomicInteger(); + + // This simulates resource allocation + @BeforeClass + public void setupAllocateResources() + { + int concurrentResources = resources.incrementAndGet(); + if (concurrentResources > 2) { + throw new IllegalStateException("Tests execute in two threads, so there should be at most 2 resources allocated, got: " + concurrentResources); + } + } + + // This simulates freeing resources + @AfterClass(alwaysRun = true) + public void tearDownReleaseResources() + { + resources.decrementAndGet(); + } + + @Test + public void test1() + { + sleepShortly(); + } + + @Test + public void test2() + { + sleepShortly(); + } + + @Test + public void test3() + { + sleepShortly(); + } + + // Sleep random time to let tests interleave. Keep sleep short not to extend tests duration too much. + private void sleepShortly() + { + try { + Thread.sleep(ThreadLocalRandom.current().nextInt(3)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass1.java b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass1.java new file mode 100644 index 0000000000..34c524ca67 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass1.java @@ -0,0 +1,3 @@ +package testng.parallelOrdering; + +public class TestClass1 extends Base {} diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass2.java b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass2.java new file mode 100644 index 0000000000..8b11761ded --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass2.java @@ -0,0 +1,3 @@ +package testng.parallelOrdering; + +public class TestClass2 extends Base {} diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass3.java b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass3.java new file mode 100644 index 0000000000..b5ea787d24 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass3.java @@ -0,0 +1,3 @@ +package testng.parallelOrdering; + +public class TestClass3 extends Base {} diff --git a/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass4.java b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass4.java new file mode 100644 index 0000000000..51b9fc51ac --- /dev/null +++ b/surefire-its/src/test/resources/surefire-1967-testng-method-parallel-ordering/src/test/java/testng/parallelOrdering/TestClass4.java @@ -0,0 +1,3 @@ +package testng.parallelOrdering; + +public class TestClass4 extends Base {} diff --git a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGExecutor.java b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGExecutor.java index 56fc9d5987..6c4b183e9a 100644 --- a/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGExecutor.java +++ b/surefire-providers/surefire-testng/src/main/java/org/apache/maven/surefire/testng/TestNGExecutor.java @@ -72,11 +72,26 @@ final class TestNGExecutor private static final boolean HAS_TEST_ANNOTATION_ON_CLASSPATH = tryLoadClass( TestNGExecutor.class.getClassLoader(), "org.testng.annotations.Test" ) != null; + // Using reflection because XmlClass.m_index is not available in older versions of TestNG + private static final Method XML_CLASS_SET_INDEX = findXmlClassSetIndexMethod(); + private TestNGExecutor() { throw new IllegalStateException( "not instantiable constructor" ); } + private static Method findXmlClassSetIndexMethod() + { + try + { + return XmlClass.class.getMethod( "setIndex", int.class ); + } + catch ( NoSuchMethodException e ) + { + return null; + } + } + @SuppressWarnings( "checkstyle:parameternumbercheck" ) static void run( Iterable> testClasses, String testSourceDirectory, Map options, // string,string because TestNGMapConfigurator#configure() @@ -127,7 +142,7 @@ static void run( Iterable> testClasses, String testSourceDirectory, suiteAndNamedTests.testNameToTest.put( metadata.testName, xmlTest ); } - xmlTest.getXmlClasses().add( new XmlClass( testClass.getName() ) ); + addXmlClass( xmlTest.getXmlClasses(), testClass.getName() ); } testng.setXmlSuites( xmlSuites ); @@ -137,6 +152,31 @@ static void run( Iterable> testClasses, String testSourceDirectory, testng.run(); } + private static void addXmlClass( List xmlClasses, String testClassName ) + { + XmlClass xmlClass = new XmlClass( testClassName ); + if ( XML_CLASS_SET_INDEX != null ) + { + try + { + // In case of parallel test execution with parallel="methods", TestNG orders test execution + // by XmlClass.m_index field. When unset (equal for all XmlClass instances), TestNG can + // invoke `@BeforeClass` setup methods on many instances, without invoking `@AfterClass` + // tearDown methods, thus leading to high resource consumptions when test instances + // allocate resources. + // With index set, order of setup, test and tearDown methods is reasonable, with approximately + // #thread-count many test classes being initialized at given point in time. + // Note: XmlClass.m_index field is set automatically by TestNG when it reads a suite file. + XML_CLASS_SET_INDEX.invoke( xmlClass, xmlClasses.size() ); + } + catch ( ReflectiveOperationException e ) + { + throw new RuntimeException( e ); + } + } + xmlClasses.add( xmlClass ); + } + private static boolean isCliDebugOrShowErrors( List mainCliOptions ) { return mainCliOptions.contains( LOGGING_LEVEL_DEBUG ) || mainCliOptions.contains( SHOW_ERRORS );