From a30dc42ef2e5343a8816b1483d3270ac74a3d4d2 Mon Sep 17 00:00:00 2001 From: bj-9527 Date: Thu, 24 Feb 2022 16:36:46 +0800 Subject: [PATCH] configfailurepolicy=continue only works for BeforeTest when using TestNG XML file Fixes #2731 --- CHANGES.txt | 1 + .../internal/invokers/ConfigInvoker.java | 3 +- .../testng/internal/invokers/TestInvoker.java | 3 +- .../FailureContinuePolicyTest.java | 182 ++++++++++++++++++ .../FailurePolicyTest.java | 128 +++++------- .../issue2731/ConfigFailTestSample.java | 33 ++++ .../test/listeners/github1602/IssueTest.java | 12 +- .../test/listeners/issue1777/IssueTest.java | 2 +- testng-core/src/test/resources/testng.xml | 1 + 9 files changed, 278 insertions(+), 87 deletions(-) create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java diff --git a/CHANGES.txt b/CHANGES.txt index 4556cc3a23..7b128bd6e8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ Current 7.6.0 +Fixed: GITHUB-2731: configfailurepolicy=continue only works for BeforeTest when using TestNG XML file (Nan Liang) Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang) Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek) Fixed: GITHUB-2709: Testnames not working together with suites in suite (Martin Aldrin) diff --git a/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java index a778b2111d..11d4a1b519 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java @@ -274,7 +274,8 @@ public void invokeConfigurations(ConfigMethodArguments arguments) { tm.getGroups(), arguments.getTestClass(), arguments.getInstance()) - && !alwaysRun) { + && !alwaysRun + && !m_continueOnFailedConfiguration) { log(3, "Skipping " + Utils.detailedMethodName(tm, true)); InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), testResult); runConfigurationListeners(testResult, arguments.getTestMethod(), true /* before */); diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java index f4a4ce4262..15c78b73e5 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java @@ -597,7 +597,8 @@ private ITestResult invokeMethod( arguments.getTestMethod(), arguments.getTestMethod().getGroups(), arguments.getTestClass(), - arguments.getInstance())) { + arguments.getInstance()) + && suite.getConfigFailurePolicy() == XmlSuite.FailurePolicy.SKIP) { Throwable exception = ExceptionUtils.getExceptionDetails(m_testContext, arguments.getInstance()); ITestResult result = diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java b/testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java new file mode 100644 index 0000000000..5e475ef497 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java @@ -0,0 +1,182 @@ +package test.configurationfailurepolicy; + +import org.testng.ITestContext; +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.xml.XmlSuite; +import test.configurationfailurepolicy.issue2731.ConfigFailTestSample; +import testhelper.OutputDirectoryPatch; + +import static org.testng.Assert.assertEquals; +import static test.SimpleBaseTest.getPathToResource; + +public class FailureContinuePolicyTest { + // only if this is run from an xml file that sets this on the suite + @BeforeClass(enabled = false) + public void setupClass(ITestContext testContext) { + assertEquals( + testContext.getSuite().getXmlSuite().getConfigFailurePolicy(), + XmlSuite.FailurePolicy.CONTINUE); + } + + @DataProvider(name = "dp") + public Object[][] getData() { + return new Object[][] { + // params - confFail, confSkip, skippedTests + new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 0, 0}, + new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 0, 0}, + new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 0}, + new Object[] { + new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class}, + 1, + 0, + 0 + }, + new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 0}, + new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 0, 0}, + new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 0, 0}, + new Object[] { + new Class[] { + ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class + }, + 2, + 0, + 0 + }, + new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 0}, + new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 0}, + new Object[] { + new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 0 + }, + new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 0, 0}, + new Object[] {new Class[] {ConfigFailTestSample.class}, 4, 0, 0} + }; + } + + @Test(dataProvider = "dp") + public void confFailureTest( + Class[] classesUnderTest, + int configurationFailures, + int configurationSkips, + int skippedTests) { + + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG testng = new TestNG(); + testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory()); + testng.setTestClasses(classesUnderTest); + testng.addListener(tla); + testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); + testng.run(); + + verify(tla, configurationFailures, configurationSkips, skippedTests); + } + + @Test + public void confFailureTestInvolvingGroups() { + Class[] classesUnderTest = + new Class[] {ClassWithFailedBeforeClassMethodAndBeforeGroupsAfterClassAfterGroups.class}; + + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG testng = new TestNG(); + testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory()); + testng.setTestClasses(classesUnderTest); + testng.addListener(tla); + testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); + testng.setGroups("group1"); + testng.run(); + verify(tla, 1, 0, 0); + } + + @Test + public void commandLineTest_policyAsSkip() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + "-testclass", + ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + verify(tla, 1, 1, 2); + } + + @Test + public void commandLineTest_policyAsContinue() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "continue", + "-testclass", + ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + verify(tla, 2, 0, 0); + } + + @Test + public void commandLineTestWithXMLFile_policyAsSkip() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + getPathToResource("testng-configfailure.xml") + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + verify(tla, 1, 1, 2); + } + + @Test + public void commandLineTestWithXMLFile_policyAsContinue() { + String[] argv = + new String[] { + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "continue", + getPathToResource("testng-configfailure.xml") + }; + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG.privateMain(argv, tla); + + verify(tla, 2, 0, 0); + } + + private void verify( + TestListenerAdapter tla, + int configurationFailures, + int configurationSkips, + int skippedTests) { + assertEquals( + tla.getConfigurationFailures().size(), + configurationFailures, + "wrong number of configuration failures"); + assertEquals( + tla.getConfigurationSkips().size(), + configurationSkips, + "wrong number of configuration skips"); + assertEquals(tla.getSkippedTests().size(), skippedTests, "wrong number of skipped tests"); + } +} diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java b/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java index 329fe4248a..b39164b959 100644 --- a/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java +++ b/testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java @@ -10,6 +10,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.xml.XmlSuite; +import test.configurationfailurepolicy.issue2731.ConfigFailTestSample; import testhelper.OutputDirectoryPatch; public class FailurePolicyTest { @@ -19,39 +20,39 @@ public class FailurePolicyTest { public void setupClass(ITestContext testContext) { assertEquals( testContext.getSuite().getXmlSuite().getConfigFailurePolicy(), - XmlSuite.FailurePolicy.CONTINUE); + XmlSuite.FailurePolicy.SKIP); } @DataProvider(name = "dp") public Object[][] getData() { return new Object[][] { - // params - confFail, confSkip, skippedTests - new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1}, - new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1}, - new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2}, - new Object[] { - new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class}, - 1, - 3, - 1 - }, - new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4}, - new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2}, - new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2}, - new Object[] { - new Class[] { - ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class + // params - confFail, confSkip, skippedTests + new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1}, + new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1}, + new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2}, + new Object[] { + new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class}, + 1, + 3, + 1 }, - 2, - 3, - 3 - }, - new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1}, - new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2}, - new Object[] { - new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8 - }, - new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2}, + new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4}, + new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2}, + new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2}, + new Object[] { + new Class[] { + ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class + }, + 2, + 3, + 3 + }, + new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1}, + new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2}, + new Object[] { + new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8 + }, + new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2}, }; } @@ -67,7 +68,7 @@ public void confFailureTest( testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory()); testng.setTestClasses(classesUnderTest); testng.addListener(tla); - testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); + testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP); testng.run(); verify(tla, configurationFailures, configurationSkips, skippedTests); @@ -83,7 +84,7 @@ public void confFailureTestInvolvingGroups() { testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory()); testng.setTestClasses(classesUnderTest); testng.addListener(tla); - testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE); + testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP); testng.setGroups("group1"); testng.run(); verify(tla, 1, 3, 1); @@ -93,14 +94,14 @@ public void confFailureTestInvolvingGroups() { public void commandLineTest_policyAsSkip() { String[] argv = new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "skip", - "-testclass", - ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + "-testclass", + ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() }; TestListenerAdapter tla = new TestListenerAdapter(); TestNG.privateMain(argv, tla); @@ -108,36 +109,17 @@ public void commandLineTest_policyAsSkip() { verify(tla, 1, 1, 2); } - @Test - public void commandLineTest_policyAsContinue() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "continue", - "-testclass", - ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName() - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 2, 0, 2); - } - @Test public void commandLineTestWithXMLFile_policyAsSkip() { String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "skip", - getPathToResource("testng-configfailure.xml") + new String[]{ + "-log", + "0", + "-d", + OutputDirectoryPatch.getOutputDirectory(), + "-configfailurepolicy", + "skip", + getPathToResource("testng-configfailure.xml") }; TestListenerAdapter tla = new TestListenerAdapter(); TestNG.privateMain(argv, tla); @@ -145,24 +127,6 @@ public void commandLineTestWithXMLFile_policyAsSkip() { verify(tla, 1, 1, 2); } - @Test - public void commandLineTestWithXMLFile_policyAsContinue() { - String[] argv = - new String[] { - "-log", - "0", - "-d", - OutputDirectoryPatch.getOutputDirectory(), - "-configfailurepolicy", - "continue", - getPathToResource("testng-configfailure.xml") - }; - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG.privateMain(argv, tla); - - verify(tla, 2, 0, 2); - } - private void verify( TestListenerAdapter tla, int configurationFailures, diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java new file mode 100644 index 0000000000..0ffc3439e4 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java @@ -0,0 +1,33 @@ +package test.configurationfailurepolicy.issue2731; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +public class ConfigFailTestSample { + @BeforeSuite + public void beforeSuite() { + Assert.fail("This before suite is fail"); + } + + @BeforeTest + public void beforeTest() { + Assert.fail("This before test is fail"); + } + + @BeforeClass + public void beforeClass() { + Assert.fail("This before class is fail"); + } + + @BeforeMethod + public void beforeMethod() { + Assert.fail("This before method is fail"); + } + + @Test + public void test() {} +} diff --git a/testng-core/src/test/java/test/listeners/github1602/IssueTest.java b/testng-core/src/test/java/test/listeners/github1602/IssueTest.java index 17e883fe26..4e1e809ead 100644 --- a/testng-core/src/test/java/test/listeners/github1602/IssueTest.java +++ b/testng-core/src/test/java/test/listeners/github1602/IssueTest.java @@ -35,19 +35,27 @@ public Object[][] getData() { "BeforeInvocation_afterMethod_STARTED", "AfterInvocation_afterMethod_SUCCESS"); List baseList = + Arrays.asList( + "BeforeInvocation_beforeMethod_STARTED", + "AfterInvocation_beforeMethod_FAILURE", + "BeforeInvocation_testMethod_STARTED", + "AfterInvocation_testMethod_SUCCESS", + "BeforeInvocation_afterMethod_STARTED"); + List commonSkipList = Arrays.asList( "BeforeInvocation_beforeMethod_STARTED", "AfterInvocation_beforeMethod_FAILURE", "BeforeInvocation_testMethod_SKIP", "AfterInvocation_testMethod_SKIP", - "BeforeInvocation_afterMethod_STARTED"); + "BeforeInvocation_afterMethod_STARTED", + "AfterInvocation_afterMethod_SKIP"); List skipList = Lists.newArrayList(baseList); skipList.add("AfterInvocation_afterMethod_SKIP"); List failList = Lists.newArrayList(baseList); failList.add("AfterInvocation_afterMethod_FAILURE"); return new Object[][] { {TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, passList}, - {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, skipList}, + {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, commonSkipList}, {TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, passList}, {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, failList} }; diff --git a/testng-core/src/test/java/test/listeners/issue1777/IssueTest.java b/testng-core/src/test/java/test/listeners/issue1777/IssueTest.java index d7caf70094..b2201d43e4 100644 --- a/testng-core/src/test/java/test/listeners/issue1777/IssueTest.java +++ b/testng-core/src/test/java/test/listeners/issue1777/IssueTest.java @@ -28,7 +28,7 @@ public void testOnStartInvokedForSkippedTests() { "before_test_method: test1", "after_test_method: test1", "after_test_method: test1", - "testSkipped_test_method: test1", + "testSuccess_test_method: test1", "testStart_test_method: test2", "before_test_method: test2", "before_test_method: test2", diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index 1fb2c3dc0d..761b1cd74e 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -784,6 +784,7 @@ +