diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index c5d36de6d0e..9607d1d3f82 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -64,6 +64,35 @@ final class TestSuiteSorter */ private $cache; + /** + * @var array array A list of normalized names of tests before reordering + */ + private $originalExecutionOrder = []; + + /** + * @var array array A list of normalized names of tests affected by reordering + */ + private $executionOrder = []; + + public static function getTestSorterUID(Test $test): string + { + if ($test instanceof PhptTestCase) { + return $test->getName(); + } + + if ($test instanceof TestCase) { + $testName = $test->getName(true); + + if (\strpos($testName, '::') === false) { + $testName = \get_class($test) . '::' . $testName; + } + + return $testName; + } + + return $test->getName(); + } + public function __construct(?TestResultCacheInterface $cache = null) { $this->cache = $cache ?? new NullTestResultCache; @@ -72,7 +101,7 @@ public function __construct(?TestResultCacheInterface $cache = null) /** * @throws Exception */ - public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void + public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects, bool $isRootTestSuite = true): void { $allowedOrders = [ self::ORDER_DEFAULT, @@ -98,9 +127,13 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend ); } + if ($isRootTestSuite) { + $this->originalExecutionOrder = $this->calculateTestExecutionOrder($suite); + } + if ($suite instanceof TestSuite) { foreach ($suite as $_suite) { - $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects); + $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects, false); } if ($orderDefects === self::ORDER_DEFECTS_FIRST) { @@ -109,6 +142,20 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend $this->sort($suite, $order, $resolveDependencies, $orderDefects); } + + if ($isRootTestSuite) { + $this->executionOrder = $this->calculateTestExecutionOrder($suite); + } + } + + public function getOriginalExecutionOrder(): array + { + return $this->originalExecutionOrder; + } + + public function getExecutionOrder(): array + { + return $this->executionOrder; } private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void @@ -139,7 +186,7 @@ private function addSuiteToDefectSortOrder(TestSuite $suite): void $max = 0; foreach ($suite->tests() as $test) { - $testname = $this->getNormalizedTestName($test); + $testname = self::getTestSorterUID($test); if (!isset($this->defectSortOrder[$testname])) { $this->defectSortOrder[$testname] = self::DEFECT_SORT_WEIGHT[$this->cache->getState($testname)]; @@ -205,8 +252,8 @@ function ($left, $right) { */ private function cmpDefectPriorityAndTime(Test $a, Test $b): int { - $priorityA = $this->defectSortOrder[$this->getNormalizedTestName($a)] ?? 0; - $priorityB = $this->defectSortOrder[$this->getNormalizedTestName($b)] ?? 0; + $priorityA = $this->defectSortOrder[self::getTestSorterUID($a)] ?? 0; + $priorityB = $this->defectSortOrder[self::getTestSorterUID($b)] ?? 0; if ($priorityB <=> $priorityA) { // Sort defect weight descending @@ -226,7 +273,7 @@ private function cmpDefectPriorityAndTime(Test $a, Test $b): int */ private function cmpDuration(Test $a, Test $b): int { - return $this->cache->getTime($this->getNormalizedTestName($a)) <=> $this->cache->getTime($this->getNormalizedTestName($b)); + return $this->cache->getTime(self::getTestSorterUID($a)) <=> $this->cache->getTime(self::getTestSorterUID($b)); } /** @@ -252,7 +299,7 @@ private function resolveDependencies(array $tests): array do { $todoNames = \array_map( function ($test) { - return $this->getNormalizedTestName($test); + return self::getTestSorterUID($test); }, $tests ); @@ -268,28 +315,6 @@ function ($test) { return \array_merge($newTestOrder, $tests); } - /** - * @param DataProviderTestSuite|TestCase $test - * - * @return string Full test name as "TestSuiteClassName::testMethodName" - */ - private function getNormalizedTestName($test): string - { - if ($test instanceof TestSuite && !($test instanceof DataProviderTestSuite)) { - return $test->getName(); - } - - if ($test instanceof PhptTestCase) { - return $test->getName(); - } - - if (\strpos($test->getName(), '::') !== false) { - return $test->getName(true); - } - - return \get_class($test) . '::' . $test->getName(true); - } - /** * @param DataProviderTestSuite|TestCase $test * @@ -312,4 +337,21 @@ function ($name) use ($testClass) { return $names; } + + private function calculateTestExecutionOrder(Test $suite): array + { + $tests = []; + + if ($suite instanceof TestSuite) { + foreach ($suite->tests() as $test) { + if (!($test instanceof TestSuite)) { + $tests[] = self::getTestSorterUID($test); + } else { + $tests = \array_merge($tests, $this->calculateTestExecutionOrder($test)); + } + } + } + + return $tests; + } } diff --git a/src/TextUI/TestRunner.php b/src/TextUI/TestRunner.php index bc167dd14e7..69cfa96ad97 100644 --- a/src/TextUI/TestRunner.php +++ b/src/TextUI/TestRunner.php @@ -39,6 +39,7 @@ use PHPUnit\Util\Log\JUnit; use PHPUnit\Util\Log\TeamCity; use PHPUnit\Util\Printer; +use PHPUnit\Util\TestDox\CliTestDoxPrinter; use PHPUnit\Util\TestDox\HtmlResultPrinter; use PHPUnit\Util\TestDox\TextResultPrinter; use PHPUnit\Util\TestDox\XmlResultPrinter; @@ -203,6 +204,7 @@ public function doRun(Test $suite, array $arguments = [], bool $exit = true): Te $sorter = new TestSuiteSorter($cache); $sorter->reorderTestsInSuite($suite, $arguments['executionOrder'], $arguments['resolveDependencies'], $arguments['executionOrderDefects']); + $originalExecutionOrder = $sorter->getOriginalExecutionOrder(); unset($sorter); } @@ -309,6 +311,11 @@ public function doRun(Test $suite, array $arguments = [], bool $exit = true): Te $arguments['columns'], $arguments['reverseList'] ); + + if (isset($originalExecutionOrder) && ($this->printer instanceof CliTestDoxPrinter)) { + /* @var CliTestDoxPrinter */ + $this->printer->setOriginalExecutionOrder($originalExecutionOrder); + } } } diff --git a/src/Util/TestDox/CliTestDoxPrinter.php b/src/Util/TestDox/CliTestDoxPrinter.php index 209ae255261..124b781dd1a 100644 --- a/src/Util/TestDox/CliTestDoxPrinter.php +++ b/src/Util/TestDox/CliTestDoxPrinter.php @@ -16,8 +16,8 @@ use PHPUnit\Framework\TestSuite; use PHPUnit\Framework\Warning; use PHPUnit\Runner\PhptTestCase; +use PHPUnit\Runner\TestSuiteSorter; use PHPUnit\TextUI\ResultPrinter; -use PHPUnit\Util\TestDox\TestResult as TestDoxTestResult; use SebastianBergmann\Timer\Timer; /** @@ -27,42 +27,96 @@ class CliTestDoxPrinter extends ResultPrinter { /** - * @var TestDoxTestResult + * @var int[] */ - private $currentTestResult; + private $nonSuccessfulTestResults = []; /** - * @var TestDoxTestResult + * @var NamePrettifier */ - private $previousTestResult; + private $prettifier; /** - * @var TestDoxTestResult[] + * @var int The number of test results received from the TestRunner */ - private $nonSuccessfulTestResults = []; + private $testIndex = 0; /** - * @var NamePrettifier + * @var int The number of test results already sent to the output */ - private $prettifier; + private $testFlushIndex = 0; - public function __construct($out = null, bool $verbose = false, $colors = self::COLOR_DEFAULT, bool $debug = false, $numberOfColumns = 80, bool $reverse = false) - { + /** + * @var array Buffer for write() + */ + private $outputBuffer = []; + + /** + * @var bool + */ + private $bufferExecutionOrder = false; + + /** + * @var array array + */ + private $originalExecutionOrder = []; + + /** + * @var string Classname of the current test + */ + private $className = ''; + + /** + * @var string Classname of the previous test; empty for first test + */ + private $lastClassName = ''; + + /** + * @var string Prettified test name of current test + */ + private $testMethod; + + /** + * @var string Test result message of current test + */ + private $testResultMessage; + + /** + * @var bool Test result message of current test contains a verbose dump + */ + private $lastFlushedTestWasVerbose = false; + + public function __construct( + $out = null, + bool $verbose = false, + $colors = self::COLOR_DEFAULT, + bool $debug = false, + $numberOfColumns = 80, + bool $reverse = false + ) { parent::__construct($out, $verbose, $colors, $debug, $numberOfColumns, $reverse); $this->prettifier = new NamePrettifier; } + public function setOriginalExecutionOrder(array $order): void + { + $this->originalExecutionOrder = $order; + $this->bufferExecutionOrder = !empty($order); + } + public function startTest(Test $test): void { if (!$test instanceof TestCase && !$test instanceof PhptTestCase && !$test instanceof TestSuite) { return; } - $class = \get_class($test); + $this->lastTestFailed = false; + $this->lastClassName = $this->className; + $this->testResultMessage = ''; if ($test instanceof TestCase) { - $className = $this->prettifier->prettifyTestClass($class); + $className = $this->prettifier->prettifyTestClass(\get_class($test)); $testMethod = $this->prettifier->prettifyTestCase($test); } elseif ($test instanceof TestSuite) { $className = $test->getName(); @@ -71,17 +125,12 @@ public function startTest(Test $test): void $test->getName() ); } elseif ($test instanceof PhptTestCase) { - $className = $class; + $className = \get_class($test); $testMethod = $test->getName(); } - $this->currentTestResult = new TestDoxTestResult( - function (string $color, string $buffer) { - return $this->formatWithColor($color, $buffer); - }, - $className, - $testMethod - ); + $this->className = $className; + $this->testMethod = $testMethod; parent::startTest($test); } @@ -92,70 +141,120 @@ public function endTest(Test $test, float $time): void return; } - parent::endTest($test, $time); - - $this->currentTestResult->setRuntime($time); + if ($test instanceof TestCase || $test instanceof PhptTestCase) { + $this->testIndex++; + } - $this->write($this->currentTestResult->toString($this->previousTestResult, $this->verbose)); + if ($this->lastTestFailed) { + $resultMessage = $this->testResultMessage; + $this->nonSuccessfulTestResults[] = $this->testIndex; + } else { + $resultMessage = $this->formatTestResultMessage( + $this->formatWithColor('fg-green', '✔'), + '', + $time, + $this->verbose + ); + } - $this->previousTestResult = $this->currentTestResult; + if ($this->bufferExecutionOrder) { + $this->bufferTestResult($test, $resultMessage); + $this->flushOutputBuffer(); + } else { + $this->writeTestResult($resultMessage); - if (!$this->currentTestResult->isTestSuccessful()) { - $this->nonSuccessfulTestResults[] = $this->currentTestResult; + if ($this->lastTestFailed) { + $this->bufferTestResult($test, $resultMessage); + } } + + parent::endTest($test, $time); } public function addError(Test $test, \Throwable $t, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-yellow', '✘'), - (string) $t + (string) $t, + $time, + true ); } public function addWarning(Test $test, Warning $e, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-yellow', '✘'), - (string) $e + (string) $e, + $time, + true ); } public function addFailure(Test $test, AssertionFailedError $e, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-red', '✘'), - (string) $e + (string) $e, + $time, + true ); } public function addIncompleteTest(Test $test, \Throwable $t, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-yellow', '∅'), (string) $t, - true + $time, + false ); } public function addRiskyTest(Test $test, \Throwable $t, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-yellow', '☢'), (string) $t, - true + $time, + false ); } public function addSkippedTest(Test $test, \Throwable $t, float $time): void { - $this->currentTestResult->fail( + $this->lastTestFailed = true; + $this->testResultMessage = $this->formatTestResultMessage( $this->formatWithColor('fg-yellow', '→'), (string) $t, - true + $time, + false ); } + public function bufferTestResult(Test $test, string $msg): void + { + $this->outputBuffer[$this->testIndex] = [ + 'className' => $this->className, + 'testName' => TestSuiteSorter::getTestSorterUID($test), + 'testMethod' => $this->testMethod, + 'message' => $msg, + 'failed' => $this->lastTestFailed, + 'verbose' => $this->lastFlushedTestWasVerbose, + ]; + } + + public function writeTestResult(string $msg): void + { + $msg = $this->formatTestSuiteHeader($this->lastClassName, $this->className, $msg); + $this->write($msg); + } + public function writeProgress(string $progress): void { } @@ -178,26 +277,159 @@ protected function printHeader(): void $this->write("\n" . Timer::resourceUsage() . "\n\n"); } - private function printNonSuccessfulTestsSummary(int $numberOfExecutedTests): void + private function flushOutputBuffer(): void + { + if ($this->testFlushIndex === $this->testIndex) { + return; + } + + if ($this->testFlushIndex > 0) { + $prevResult = $this->getTestResultByName($this->originalExecutionOrder[$this->testFlushIndex - 1]); + } else { + $prevResult = $this->getEmptyTestResult(); + } + + do { + $flushed = false; + $result = $this->getTestResultByName($this->originalExecutionOrder[$this->testFlushIndex]); + + if (!empty($result)) { + $this->writeBufferTestResult($prevResult, $result); + $this->testFlushIndex++; + $prevResult = $result; + $flushed = true; + } + } while ($flushed && $this->testFlushIndex < $this->testIndex); + } + + private function writeBufferTestResult(array $prevResult, array $result): void + { + // Write spacer line for new suite headers and after verbose messages + if ($prevResult['testName'] !== '' && + ($prevResult['verbose'] === true || $prevResult['className'] !== $result['className'])) { + $this->write("\n"); + } + + // Write suite header + if ($prevResult['className'] !== $result['className']) { + $this->write($result['className'] . "\n"); + } + + // Write the test result itself + $this->write($result['message']); + } + + private function getTestResultByName(string $testName): array + { + foreach ($this->outputBuffer as $result) { + if ($result['testName'] === $testName) { + return $result; + } + } + + return []; + } + + private function formatTestSuiteHeader(?string $lastClassName, string $className, string $msg): string + { + if ($lastClassName === null || $className !== $lastClassName) { + return \sprintf( + "%s%s\n%s", + ($this->lastClassName !== '') ? "\n" : '', + $className, + $msg + ); + } + + return $msg; + } + + private function formatTestResultMessage( + string $symbol, + string $resultMessage, + float $time, + bool $alwaysVerbose = false + ): string { + $additionalInformation = $this->getFormattedAdditionalInformation($resultMessage, $alwaysVerbose); + $msg = \sprintf( + " %s %s%s\n%s", + $symbol, + $this->testMethod, + $this->verbose ? ' ' . $this->getFormattedRuntime($time) : '', + $additionalInformation + ); + + $this->lastFlushedTestWasVerbose = !empty($additionalInformation); + + return $msg; + } + + private function getFormattedRuntime(float $time): string + { + if ($time > 5) { + return $this->formatWithColor('fg-red', \sprintf('[%.2f ms]', $time * 1000)); + } + + if ($time > 1) { + return $this->formatWithColor('fg-yellow', \sprintf('[%.2f ms]', $time * 1000)); + } + + return \sprintf('[%.2f ms]', $time * 1000); + } + + private function getFormattedAdditionalInformation(string $resultMessage, bool $verbose): string { - $numberOfNonSuccessfulTests = \count($this->nonSuccessfulTestResults); + if ($resultMessage === '') { + return ''; + } + + if (!($this->verbose || $verbose)) { + return ''; + } + + return \sprintf( + " │\n%s\n", + \implode( + "\n", + \array_map( + function (string $text) { + return \sprintf(' │ %s', $text); + }, + \explode("\n", $resultMessage) + ) + ) + ); + } - if ($numberOfNonSuccessfulTests === 0) { + private function printNonSuccessfulTestsSummary(int $numberOfExecutedTests): void + { + if (empty($this->nonSuccessfulTestResults)) { return; } - if (($numberOfNonSuccessfulTests / $numberOfExecutedTests) >= 0.7) { + if ((\count($this->nonSuccessfulTestResults) / $numberOfExecutedTests) >= 0.7) { return; } $this->write("Summary of non-successful tests:\n\n"); - $previousTestResult = null; - - foreach ($this->nonSuccessfulTestResults as $testResult) { - $this->write($testResult->toString($previousTestResult, $this->verbose)); + $prevResult = $this->getEmptyTestResult(); - $previousTestResult = $testResult; + foreach ($this->nonSuccessfulTestResults as $testIndex) { + $result = $this->outputBuffer[$testIndex]; + $this->writeBufferTestResult($prevResult, $result); + $prevResult = $result; } } + + private function getEmptyTestResult(): array + { + return [ + 'className' => '', + 'testName' => '', + 'message' => '', + 'failed' => '', + 'verbose' => '', + ]; + } } diff --git a/src/Util/TestResultCache.php b/src/Util/TestResultCache.php index c4fa0e3cbc2..8cb290091c8 100644 --- a/src/Util/TestResultCache.php +++ b/src/Util/TestResultCache.php @@ -9,6 +9,8 @@ */ namespace PHPUnit\Runner; +use PHPUnit\Framework\Test; + class TestResultCache implements \Serializable, TestResultCacheInterface { /** diff --git a/tests/end-to-end/regression/GitHub/3380/issue-3380-test.phpt b/tests/end-to-end/regression/GitHub/3380/issue-3380-test.phpt new file mode 100644 index 00000000000..0aca1c4f13d --- /dev/null +++ b/tests/end-to-end/regression/GitHub/3380/issue-3380-test.phpt @@ -0,0 +1,63 @@ +--TEST-- +https://github.com/sebastianbergmann/phpunit/issues/3380 +--FILE-- +assertSame([], $suite->tests()); - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, $orderDefects); - $this->assertSame([], $suite->tests()); + $this->assertEmpty($suite->tests()); + $this->assertEmpty($sorter->getOriginalExecutionOrder()); + $this->assertEmpty($sorter->getExecutionOrder()); } /** * @dataProvider commonSorterOptionsProvider */ - public function testBasicExecutionOrderOptions(int $order, bool $resolveDependencies, array $expected): void + public function testBasicExecutionOrderOptions(int $order, bool $resolveDependencies, array $expectedOrder): void { $suite = new TestSuite; $suite->addTestSuite(\MultiDependencyTest::class); @@ -72,7 +80,8 @@ public function testBasicExecutionOrderOptions(int $order, bool $resolveDependen $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFAULT); - $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + $this->assertSame(self::MULTIDEPENDENCYTEST_EXECUTION_ORDER, $sorter->getOriginalExecutionOrder()); + $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); } public function testCanSetRandomizationWithASeed(): void @@ -84,7 +93,15 @@ public function testCanSetRandomizationWithASeed(): void \mt_srand(54321); $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, false, TestSuiteSorter::ORDER_DEFAULT); - $this->assertSame(['testTwo', 'testFour', 'testFive', 'testThree', 'testOne'], $this->getTestExecutionOrder($suite)); + $expectedOrder = [ + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testOne', + ]; + + $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); } public function testCanSetRandomizationWithASeedAndResolveDependencies(): void @@ -96,7 +113,15 @@ public function testCanSetRandomizationWithASeedAndResolveDependencies(): void \mt_srand(54321); $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, true, TestSuiteSorter::ORDER_DEFAULT); - $this->assertSame(['testTwo', 'testFive', 'testOne', 'testThree', 'testFour'], $this->getTestExecutionOrder($suite)); + $expectedOrder = [ + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ]; + + $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); } /** @@ -117,7 +142,7 @@ public function testOrderDurationWithoutCache(bool $resolveDependencies, array $ TestSuiteSorter::ORDER_DEFAULT ); - $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + $this->assertSame($expected, $sorter->getExecutionOrder()); } public function orderDurationWithoutCacheProvider(): array @@ -126,21 +151,21 @@ public function orderDurationWithoutCacheProvider(): array 'dependency-ignore' => [ self::IGNORE_DEPENDENCIES, [ - 'testOne', - 'testTwo', - 'testThree', - 'testFour', - 'testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', ], ], 'dependency-resolve' => [ self::RESOLVE_DEPENDENCIES, [ - 'testOne', - 'testTwo', - 'testThree', - 'testFour', - 'testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', ], ], ]; @@ -170,7 +195,7 @@ public function testOrderDurationWithCache(bool $resolveDependencies, array $tes TestSuiteSorter::ORDER_DEFAULT ); - $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + $this->assertSame($expected, $sorter->getExecutionOrder()); } public function orderDurationWithCacheProvider(): array @@ -186,11 +211,11 @@ public function orderDurationWithCacheProvider(): array 'testFive' => 1, ], [ - 'testOne', - 'testTwo', - 'testThree', - 'testFour', - 'testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', ], ], 'duration-same-dependency-resolve' => [ @@ -203,11 +228,11 @@ public function orderDurationWithCacheProvider(): array 'testFive' => 1, ], [ - 'testOne', - 'testTwo', - 'testThree', - 'testFour', - 'testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', ], ], 'duration-different-dependency-ignore' => [ @@ -220,11 +245,11 @@ public function orderDurationWithCacheProvider(): array 'testFive' => 2, ], [ - 'testFour', - 'testFive', - 'testTwo', - 'testThree', - 'testOne', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testOne', ], ], 'duration-different-dependency-resolve' => [ @@ -237,11 +262,11 @@ public function orderDurationWithCacheProvider(): array 'testFive' => 2, ], [ - 'testFive', - 'testTwo', - 'testOne', - 'testThree', - 'testFour', + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', ], ], ]; @@ -265,7 +290,7 @@ public function testSuiteSorterDefectsOptions(int $order, bool $resolveDependenc $sorter = new TestSuiteSorter($cache); $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFECTS_FIRST); - $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + $this->assertSame($expected, $sorter->getExecutionOrder()); } /** @@ -282,28 +307,52 @@ public function commonSorterOptionsProvider(): array 'default' => [ TestSuiteSorter::ORDER_DEFAULT, self::IGNORE_DEPENDENCIES, - ['testOne', 'testTwo', 'testThree', 'testFour', 'testFive'], + [ + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], ], // Activating dependency resolution should have no effect under normal circumstances 'resolve default' => [ TestSuiteSorter::ORDER_DEFAULT, self::RESOLVE_DEPENDENCIES, - ['testOne', 'testTwo', 'testThree', 'testFour', 'testFive'], + [ + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], ], // Reversing without checks should give a simple reverse order 'reverse' => [ TestSuiteSorter::ORDER_REVERSED, self::IGNORE_DEPENDENCIES, - ['testFive', 'testFour', 'testThree', 'testTwo', 'testOne'], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + ], ], // Reversing with resolution still allows testFive to move to front, testTwo before testOne 'resolve reverse' => [ TestSuiteSorter::ORDER_REVERSED, self::RESOLVE_DEPENDENCIES, - ['testFive', 'testTwo', 'testOne', 'testThree', 'testFour'], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], ], ]; } @@ -330,7 +379,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], ], - ['testOne', 'testTwo', 'testThree', 'testFour', 'testFive'], ], + [ + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], + ], // Running with an empty cache should not spook the TestSuiteSorter 'default, empty result cache' => [ @@ -339,7 +395,14 @@ public function defectsSorterOptionsProvider(): array [ // empty result cache ], - ['testOne', 'testTwo', 'testThree', 'testFour', 'testFive'], ], + [ + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], + ], // testFive is independent and can be moved to the front 'default, testFive skipped' => [ @@ -352,7 +415,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_SKIPPED, 'time' => 1], ], - ['testFive', 'testOne', 'testTwo', 'testThree', 'testFour'], ], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], + ], // Defects in testFive and testTwo, but the faster testFive should be run first 'default, testTwo testFive skipped' => [ @@ -365,7 +435,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_SKIPPED, 'time' => 0], ], - ['testFive', 'testTwo', 'testOne', 'testThree', 'testFour'], ], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], + ], // Skipping testThree will move it to the front when ignoring dependencies 'default, testThree skipped' => [ @@ -378,7 +455,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], ], - ['testThree', 'testOne', 'testTwo', 'testFour', 'testFive'], ], + [ + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], + ], // Skipping testThree will move it to the front but behind its dependencies 'default resolve, testThree skipped' => [ @@ -391,7 +475,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], ], - ['testOne', 'testTwo', 'testThree', 'testFour', 'testFive'], ], + [ + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testFive', + ], + ], // Skipping testThree will move it to the front and keep the others reversed 'reverse, testThree skipped' => [ @@ -404,7 +495,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], ], - ['testThree', 'testFive', 'testFour', 'testTwo', 'testOne'], ], + [ + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testFour', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + ], + ], // Demonstrate a limit of the dependency resolver: after sorting defects to the front, // the resolver will mark testFive done before testThree because of dependencies @@ -418,7 +516,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_SKIPPED, 'time' => 1], ], - ['testFive', 'testOne', 'testTwo', 'testThree', 'testFour'], ], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], + ], // Torture test // - incomplete TestResultCache @@ -433,7 +538,14 @@ public function defectsSorterOptionsProvider(): array 'testTwo' => ['state' => BaseTestRunner::STATUS_PASSED, 'time' => 1], 'testThree' => ['state' => BaseTestRunner::STATUS_SKIPPED, 'time' => 1], ], - ['testFive', 'testTwo', 'testOne', 'testThree', 'testFour'], ], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], + ], // Make sure the dependency resolver is not confused by failing tests. // Scenario: Four has a @depends on Three and fails. Result: Three is still run first @@ -448,7 +560,14 @@ public function defectsSorterOptionsProvider(): array 'testFour' => ['state' => BaseTestRunner::STATUS_FAILURE, 'time' => 1], 'testFive' => ['state' => BaseTestRunner::STATUS_FAILURE, 'time' => 1], ], - ['testFive', 'testOne', 'testTwo', 'testThree', 'testFour'], ], + [ + \MultiDependencyTest::class . '::testFive', + \MultiDependencyTest::class . '::testOne', + \MultiDependencyTest::class . '::testTwo', + \MultiDependencyTest::class . '::testThree', + \MultiDependencyTest::class . '::testFour', + ], + ], ]; } @@ -486,11 +605,4 @@ public function suiteSorterOptionPermutationsProvider(): array return $data; } - - private function getTestExecutionOrder(TestSuite $suite): array - { - return \array_map(function ($test) { - return $test->getName(); - }, $suite->tests()[0]->tests()); - } }