[Performance] Cache phpunit results and run defects first to speed up the Mutation Testing execution

composer.json

@@ -49,7 +49,7 @@
         "composer/xdebug-handler": "^2.0",
         "infection/abstract-testframework-adapter": "^0.5.0",
         "infection/extension-installer": "^0.1.0",
-        "infection/include-interceptor": "^0.2.4",
+        "infection/include-interceptor": "^0.2.5",
         "justinrainbow/json-schema": "^5.2.10",
         "nikic/php-parser": "^4.10.3",
         "ocramius/package-versions": "^1.9.0 || ^2.0",

src/Command/RunCommand.php

@@ -312,7 +312,7 @@ protected function configure(): void
                 self::OPTION_DEBUG,
                 null,
                 InputOption::VALUE_NONE,
-                'Will not clean up Infection temporary folder'
+                'Will not clean up utility files from Infection temporary folder. Adds command lines to the logs and prints Initial Tests output to stdout.'
             )
             ->addOption(
                 self::OPTION_DRY_RUN,

src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php

@@ -37,6 +37,7 @@
 
 use Infection\Event\MutationTestingWasFinished;
 use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Component\Finder\Finder;
 
 /**
  * @internal
@@ -54,6 +55,11 @@ public function __construct(Filesystem $filesystem, string $tmpDir)
 
     public function onMutationTestingWasFinished(MutationTestingWasFinished $event): void
     {
-        $this->filesystem->remove($this->tmpDir);
+        $finder = Finder::create()
+            ->in($this->tmpDir)
+            // leave PHPUnit's result cache files so that subsequent Infection runs are faster because of `executionOrder=defects`
+            ->notName('/\.phpunit\.result\.cache\.(.*)/');
+
+        $this->filesystem->remove($finder);
     }
 }

src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php

@@ -99,12 +99,10 @@ public function build(
             $originalBootstrapFile = $this->originalBootstrapFile = $this->getOriginalBootstrapFilePath($xPath);
         }
 
-        // if original phpunit.xml has order by random, we should replace it to use `default` order and our sorting
-        // by <file> tags (e.g. the fastest tests first)
-        $this->configManipulator->setDefaultTestsOrderAttribute($version, $xPath);
+        // activate PHPUnit's result cache and order tests by running defects first, then sorted by fastest first
+        $this->configManipulator->handleResultCacheAndExecutionOrder($version, $xPath, $mutationHash);
         $this->configManipulator->setStopOnFailure($xPath);
         $this->configManipulator->deactivateColours($xPath);
-        $this->configManipulator->deactivateResultCaching($xPath);
         $this->configManipulator->deactivateStderrRedirection($xPath);
         $this->configManipulator->removeExistingLoggers($xPath);
         $this->configManipulator->removeExistingPrinters($xPath);

src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php

@@ -100,13 +100,21 @@ public function deactivateResultCaching(SafeDOMXPath $xPath): void
         $this->setAttributeValue($xPath, 'cacheResult', 'false');
     }
 
-    public function setDefaultTestsOrderAttribute(string $version, SafeDOMXPath $xPath): void
+    public function handleResultCacheAndExecutionOrder(string $version, SafeDOMXPath $xPath, string $mutationHash): void
     {
-        if (version_compare($version, '7.2', '<')) {
+        // starting from PHPUnit 7.3 we can set cache result and "defects" execution order https://github.com/sebastianbergmann/phpunit/blob/7.3.0/phpunit.xsd
+        if (version_compare($version, '7.3', '>=')) {
+            $this->setAttributeValue($xPath, 'cacheResult', 'true');
+            $this->setAttributeValue($xPath, 'cacheResultFile', sprintf('.phpunit.result.cache.%s', $mutationHash));
+            $this->setAttributeValue($xPath, 'executionOrder', 'defects');
+
             return;
         }
 
-        $this->setAttributeValue($xPath, 'executionOrder', 'default');
+        // from 7.2 to 7.3 we only can set "default" execution order and no cache result https://github.com/sebastianbergmann/phpunit/blob/7.2.0/phpunit.xsd
+        if (version_compare($version, '7.2', '>=')) {
+            $this->setAttributeValue($xPath, 'executionOrder', 'default');
+        }
     }
 
     public function deactivateStderrRedirection(SafeDOMXPath $xPath): void

tests/e2e/PestTestFramework/composer.json

@@ -1,6 +1,6 @@
 {
     "require-dev": {
-        "pestphp/pest": "^1.2.0"
+        "pestphp/pest": "^1.15.0"
     },
     "autoload": {
         "psr-4": {

tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php

@@ -135,7 +135,7 @@ public function test_it_preserves_white_spaces_and_formatting(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.a1b2c3.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.a1b2c3.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests"/>
   </testsuites>
@@ -171,7 +171,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash1.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash1.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests">
       <file>/path/to/FooTest.php</file>
@@ -241,7 +241,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash2.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash2.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests">
       <file>/path/to/BarTest.php</file>
@@ -465,7 +465,26 @@ public function test_it_removes_printer_class(): void
         $this->assertSame(0, $printerClass->length);
     }
 
-    public function test_it_sets_default_execution_order_when_attribute_is_absent(): void
+    public function test_it_does_not_set_default_execution_order_for_phpunit_7_1(): void
+    {
+        $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml');
+
+        $xml = file_get_contents(
+            $builder->build(
+                [],
+                self::MUTATED_FILE_PATH,
+                self::HASH,
+                self::ORIGINAL_FILE_PATH,
+                '7.1'
+            )
+        );
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder');
+
+        $this->assertSame(0, $executionOrder->length);
+    }
+
+    public function test_it_sets_default_execution_order_when_attribute_is_absent_for_phpunit_7_2(): void
     {
         $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml');
 
@@ -484,7 +503,7 @@ public function test_it_sets_default_execution_order_when_attribute_is_absent():
         $this->assertSame('default', $executionOrder);
     }
 
-    public function test_it_sets_default_execution_order_when_attribute_is_present(): void
+    public function test_it_sets_default_execution_order_when_attribute_is_present_for_phpunit_7_2(): void
     {
         $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml');
 
@@ -503,6 +522,30 @@ public function test_it_sets_default_execution_order_when_attribute_is_present()
         $this->assertSame('default', $executionOrder);
     }
 
+    public function test_it_sets_defects_execution_order_and_cache_result_when_attribute_is_present_for_phpunit_7_3(): void
+    {
+        $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml');
+
+        $xml = file_get_contents(
+            $builder->build(
+                [],
+                self::MUTATED_FILE_PATH,
+                self::HASH,
+                self::ORIGINAL_FILE_PATH,
+                '7.3'
+            )
+        );
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue;
+        $this->assertSame('defects', $executionOrder);
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResult')[0]->nodeValue;
+        $this->assertSame('true', $executionOrder);
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResultFile')[0]->nodeValue;
+        $this->assertSame(sprintf('.phpunit.result.cache.%s', self::HASH), $executionOrder);
+    }
+
     /**
      * @dataProvider locationsProvider
      *

tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php

@@ -600,7 +600,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x
         );
     }
 
-    public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void
+    public function test_it_activates_result_cache_and_execution_order_defects_for_phpunit_7_3(): void
     {
         $this->assertItChangesXML(<<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
@@ -611,20 +611,22 @@ public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void
 XML
             ,
             static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
-                $configManipulator->setDefaultTestsOrderAttribute('7.2', $xPath);
+                $configManipulator->handleResultCacheAndExecutionOrder('7.3', $xPath, 'a1b2c3');
             },
             <<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
 <phpunit
-    executionOrder="default"
+    executionOrder="defects"
+    cacheResult="true"
+    cacheResultFile=".phpunit.result.cache.a1b2c3"
     syntaxCheck="false"
 >
 </phpunit>
 XML
         );
     }
 
-    public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(): void
+    public function test_it_does_not_set_result_cache_for_phpunit_7_1(): void
     {
         $this->assertItChangesXML(<<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
@@ -635,7 +637,7 @@ public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(
 XML
             ,
             static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
-                $configManipulator->setDefaultTestsOrderAttribute('7.1', $xPath);
+                $configManipulator->handleResultCacheAndExecutionOrder('7.1', $xPath, 'a1b2c3');
             },
             <<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>