Implement #3748 sort tests by @Small, @Medium and @large annotation

phpunit.xsd

@@ -181,14 +181,17 @@
             <xs:enumeration value="depends,duration"/>
             <xs:enumeration value="depends,random"/>
             <xs:enumeration value="depends,reverse"/>
+            <xs:enumeration value="depends,size"/>
             <xs:enumeration value="duration"/>
             <xs:enumeration value="no-depends"/>
             <xs:enumeration value="no-depends,defects"/>
             <xs:enumeration value="no-depends,duration"/>
             <xs:enumeration value="no-depends,random"/>
             <xs:enumeration value="no-depends,reverse"/>
+            <xs:enumeration value="no-depends,size"/>
             <xs:enumeration value="random"/>
             <xs:enumeration value="reverse"/>
+            <xs:enumeration value="size"/>
         </xs:restriction>
     </xs:simpleType>
     <xs:complexType name="fileFilterType">

src/Framework/DataProviderTestSuite.php

@@ -9,6 +9,8 @@
  */
 namespace PHPUnit\Framework;
 
+use PHPUnit\Util\Test as TestUtil;
+
 /**
  * @internal This class is not covered by the backward compatibility promise for PHPUnit
  */
@@ -40,4 +42,16 @@ public function hasDependencies(): bool
     {
         return \count($this->dependencies) > 0;
     }
+
+    /**
+     * Returns the size of the each test created using the data provider(s)
+     *
+     * @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
+     */
+    public function getSize(): int
+    {
+        [$className, $methodName] = \explode('::', $this->getName());
+
+        return TestUtil::getSize($className, $methodName);
+    }
 }

src/Runner/TestSuiteSorter.php

@@ -44,6 +44,13 @@ final class TestSuiteSorter
      */
     public const ORDER_DURATION = 4;
 
+    /**
+     * Order tests by @size annotation 'small', 'medium', 'large'
+     *
+     * @var int
+     */
+    public const ORDER_SIZE = 5;
+
     /**
      * List of sorting weights for all test result codes. A higher number gives higher priority.
      */
@@ -115,11 +122,12 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend
             self::ORDER_REVERSED,
             self::ORDER_RANDOMIZED,
             self::ORDER_DURATION,
+            self::ORDER_SIZE,
         ];
 
         if (!\in_array($order, $allowedOrders, true)) {
             throw new Exception(
-                '$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION'
+                '$order must be one of TestSuiteSorter::ORDER_[DEFAULT|REVERSED|RANDOMIZED|DURATION|SIZE]'
             );
         }
 
@@ -177,6 +185,8 @@ private function sort(TestSuite $suite, int $order, bool $resolveDependencies, i
             $suite->setTests($this->randomize($suite->tests()));
         } elseif ($order === self::ORDER_DURATION && $this->cache !== null) {
             $suite->setTests($this->sortByDuration($suite->tests()));
+        } elseif ($order === self::ORDER_SIZE) {
+            $suite->setTests($this->sortBySize($suite->tests()));
         }
 
         if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) {
@@ -260,6 +270,21 @@ function ($left, $right) {
         return $tests;
     }
 
+    private function sortBySize(array $tests): array
+    {
+        \usort(
+            $tests,
+            /**
+             * @throws \SebastianBergmann\RecursionContext\InvalidArgumentException
+             */
+            function ($left, $right) {
+                return $this->cmpSize($left, $right);
+            }
+        );
+
+        return $tests;
+    }
+
     /**
      * Comparator callback function to sort tests for "reach failure as fast as possible":
      * 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT
@@ -296,6 +321,17 @@ private function cmpDuration(Test $a, Test $b): int
         return $this->cache->getTime(self::getTestSorterUID($a)) <=> $this->cache->getTime(self::getTestSorterUID($b));
     }
 
+    /**
+     * Compares test size for sorting test small->medium->large->unknown
+     */
+    private function cmpSize(Test $a, Test $b): int
+    {
+        $sizeA = ($size = $a->getSize()) !== -1 ? $size : 4;
+        $sizeB = ($size = $b->getSize()) !== -1 ? $size : 4;
+
+        return $sizeA <=> $sizeB;
+    }
+
     /**
      * Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible.
      * The algorithm will leave the tests in original running order when it can.

src/TextUI/Command.php

@@ -1302,6 +1302,11 @@ private function handleOrderByOption(string $value): void
 
                     break;
 
+                case 'size':
+                    $this->arguments['executionOrder'] = TestSuiteSorter::ORDER_SIZE;
+
+                    break;
+
                 default:
                     $this->exitWithErrorMessage("unrecognized --order-by option: $order");
             }

src/Util/Configuration.php

@@ -863,6 +863,11 @@ public function getPHPUnitConfiguration(): array
                     case 'reverse':
                         $result['executionOrder'] = TestSuiteSorter::ORDER_REVERSED;
 
+                        break;
+
+                    case 'size':
+                        $result['executionOrder'] = TestSuiteSorter::ORDER_SIZE;
+
                         break;
                 }
             }

tests/_files/TestWithDifferentSizes.php

@@ -23,6 +23,7 @@ public function testWithSizeLarge(): void
     }
 
     /**
+     * @depends testDataProviderWithSizeMedium
      * @medium
      */
     public function testWithSizeMedium(): void
@@ -35,4 +36,28 @@ public function testWithSizeMedium(): void
     public function testWithSizeSmall(): void
     {
     }
+
+    /**
+     * @dataProvider provider
+     * @small
+     */
+    public function testDataProviderWithSizeSmall(bool $value): void
+    {
+    }
+
+    /**
+     * @dataProvider provider
+     * @medium
+     */
+    public function testDataProviderWithSizeMedium(bool $value): void
+    {
+    }
+
+    public function provider(): array
+    {
+        return [
+            [false],
+            [true],
+        ];
+    }
 }

tests/unit/Runner/TestSuiteSorterTest.php

@@ -41,7 +41,7 @@ public function testThrowsExceptionWhenUsingInvalidOrderOption(): void
         $sorter = new TestSuiteSorter;
 
         $this->expectException(Exception::class);
-        $this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION');
+        $this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_[DEFAULT|REVERSED|RANDOMIZED|DURATION|SIZE]');
         $sorter->reorderTestsInSuite($suite, -1, false, TestSuiteSorter::ORDER_DEFAULT);
     }
 
@@ -608,4 +608,26 @@ public function suiteSorterOptionPermutationsProvider(): array
 
         return $data;
     }
+
+    public function testOrderBySize(): void
+    {
+        $suite = new TestSuite;
+        $suite->addTestSuite(\TestWithDifferentSizes::class);
+        $sorter = new TestSuiteSorter;
+
+        $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_SIZE, true, TestSuiteSorter::ORDER_DEFAULT);
+
+        $expectedOrder = [
+            \TestWithDifferentSizes::class . '::testWithSizeSmall',
+            \TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #0',
+            \TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #1',
+            \TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #0',
+            \TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #1',
+            \TestWithDifferentSizes::class . '::testWithSizeMedium',
+            \TestWithDifferentSizes::class . '::testWithSizeLarge',
+            \TestWithDifferentSizes::class . '::testWithSizeUnknown',
+        ];
+
+        $this->assertSame($expectedOrder, $sorter->getExecutionOrder());
+    }
 }