From 92bbfb25d881603c4eee1902247591091d6c9643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20M=C3=B6ller?= Date: Sat, 8 Sep 2018 12:30:34 +0200 Subject: [PATCH] Enhancement: Allow to order tests by time (duration) --- phpunit.xsd | 2 + src/Runner/TestSuiteSorter.php | 33 ++++- tests/unit/Runner/TestSuiteSorterTest.php | 150 +++++++++++++++++++++- 3 files changed, 181 insertions(+), 4 deletions(-) diff --git a/phpunit.xsd b/phpunit.xsd index cca0a3b412d..7eaa65eda1d 100644 --- a/phpunit.xsd +++ b/phpunit.xsd @@ -176,12 +176,14 @@ + + diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index 0716120f616..c19c0e88591 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -36,6 +36,11 @@ final class TestSuiteSorter */ public const ORDER_DEFECTS_FIRST = 3; + /** + * @var int + */ + public const ORDER_DURATION = 4; + /** * List of sorting weights for all test result codes. A higher number gives higher priority. */ @@ -73,11 +78,12 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend self::ORDER_DEFAULT, self::ORDER_REVERSED, self::ORDER_RANDOMIZED, + self::ORDER_DURATION, ]; if (!\in_array($order, $allowedOrders, true)) { throw new Exception( - '$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED' + '$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION' ); } @@ -115,6 +121,8 @@ private function sort(TestSuite $suite, int $order, bool $resolveDependencies, i $suite->setTests($this->reverse($suite->tests())); } elseif ($order === self::ORDER_RANDOMIZED) { $suite->setTests($this->randomize($suite->tests())); + } elseif ($order === self::ORDER_DURATION && $this->cache !== null) { + $suite->setTests($this->sortByDuration($suite->tests())); } if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) { @@ -175,6 +183,18 @@ function ($left, $right) { return $tests; } + private function sortByDuration(array $tests): array + { + \usort( + $tests, + function ($left, $right) { + return $this->cmpDuration($left, $right); + } + ); + + return $tests; + } + /** * Comparator callback function to sort tests for "reach failure as fast as possible": * 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT @@ -192,14 +212,21 @@ private function cmpDefectPriorityAndTime(Test $a, Test $b): int } if ($priorityA || $priorityB) { - // Sort test duration ascending - return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName()); + return $this->cmpDuration($a, $b); } // do not change execution order return 0; } + /** + * Compares test duration for sorting tests by duration ascending. + */ + private function cmpDuration(Test $a, Test $b): int + { + return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName()); + } + /** * Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible. * The algorithm will leave the tests in original running order when it can. diff --git a/tests/unit/Runner/TestSuiteSorterTest.php b/tests/unit/Runner/TestSuiteSorterTest.php index 325eb047023..6e5f745b88f 100644 --- a/tests/unit/Runner/TestSuiteSorterTest.php +++ b/tests/unit/Runner/TestSuiteSorterTest.php @@ -31,7 +31,7 @@ public function testThrowsExceptionWhenUsingInvalidOrderOption(): void $sorter = new TestSuiteSorter(); $this->expectException(Exception::class); - $this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED'); + $this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION'); $sorter->reorderTestsInSuite($suite, -1, false, TestSuiteSorter::ORDER_DEFAULT); } @@ -99,6 +99,154 @@ public function testCanSetRandomizationWithASeedAndResolveDependencies(): void $this->assertSame(['testTwo', 'testFive', 'testOne', 'testThree', 'testFour'], $this->getTestExecutionOrder($suite)); } + /** + * @dataProvider orderDurationWithoutCacheProvider + */ + public function testOrderDurationWithoutCache(bool $resolveDependencies, array $expected): void + { + $suite = new TestSuite; + + $suite->addTestSuite(\MultiDependencyTest::class); + + $sorter = new TestSuiteSorter(); + + $sorter->reorderTestsInSuite( + $suite, + TestSuiteSorter::ORDER_DURATION, + $resolveDependencies, + TestSuiteSorter::ORDER_DEFAULT + ); + + $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + } + + public function orderDurationWithoutCacheProvider(): array + { + return [ + 'dependency-ignore' => [ + self::IGNORE_DEPENDENCIES, + [ + 'testOne', + 'testTwo', + 'testThree', + 'testFour', + 'testFive', + ], + ], + 'dependency-resolve' => [ + self::RESOLVE_DEPENDENCIES, + [ + 'testOne', + 'testTwo', + 'testThree', + 'testFour', + 'testFive', + ], + ], + ]; + } + + /** + * @dataProvider orderDurationWithCacheProvider + */ + public function testOrderDurationWithCache(bool $resolveDependencies, array $testTimes, array $expected): void + { + $suite = new TestSuite; + + $suite->addTestSuite(\MultiDependencyTest::class); + + $cache = new TestResultCache(); + + foreach ($testTimes as $testName => $time) { + $cache->setTime($testName, $time); + } + + $sorter = new TestSuiteSorter($cache); + + $sorter->reorderTestsInSuite( + $suite, + TestSuiteSorter::ORDER_DURATION, + $resolveDependencies, + TestSuiteSorter::ORDER_DEFAULT + ); + + $this->assertSame($expected, $this->getTestExecutionOrder($suite)); + } + + public function orderDurationWithCacheProvider(): array + { + return [ + 'duration-same-dependency-ignore' => [ + self::IGNORE_DEPENDENCIES, + [ + 'testOne' => 1, + 'testTwo' => 1, + 'testThree' => 1, + 'testFour' => 1, + 'testFive' => 1, + ], + [ + 'testOne', + 'testTwo', + 'testThree', + 'testFour', + 'testFive', + ], + ], + 'duration-same-dependency-resolve' => [ + self::RESOLVE_DEPENDENCIES, + [ + 'testOne' => 1, + 'testTwo' => 1, + 'testThree' => 1, + 'testFour' => 1, + 'testFive' => 1, + ], + [ + 'testOne', + 'testTwo', + 'testThree', + 'testFour', + 'testFive', + ], + ], + 'duration-different-dependency-ignore' => [ + self::IGNORE_DEPENDENCIES, + [ + 'testOne' => 5, + 'testTwo' => 3, + 'testThree' => 4, + 'testFour' => 1, + 'testFive' => 2, + ], + [ + 'testFour', + 'testFive', + 'testTwo', + 'testThree', + 'testOne', + ], + ], + 'duration-different-dependency-resolve' => [ + self::RESOLVE_DEPENDENCIES, + [ + 'testOne' => 5, + 'testTwo' => 3, + 'testThree' => 4, + 'testFour' => 1, + 'testFive' => 2, + ], + [ + 'testFive', + 'testTwo', + 'testOne', + 'testThree', + 'testFour', + ], + ], + ]; + } + /** * @dataProvider defectsSorterOptionsProvider */