[Performance] Cache phpunit results and run defects first to speed up…

… the Mutation Testing execution (#1549) * Cache phpunit results and run defects first to speed up the Mutation Testing execution When Infection runs Mutation Analysis for the first time, we can save the information about which Mutant was killed by which test, and then on the subsequent Infection execution (second and more), we can use this cached knowledge and run those test first that killed particular Mutant in a previous call. This should speed up MT as killed Mutants will be killed from the first "shot". * Debug why Pest e2e test fails * Dump the log file for Pest e2e test * Log killed mutants for Pest e2e test * Update `infection/include-interceptor` * Revert debug changes in Pest e2e test * Remove debug `cat` * Set the minimum version of `infection/include-interceptor` - `0.2.5`
infection · Sep 4, 2021 · bba07d8 · bba07d8
1 parent c87fab9
commit bba07d8
Show file tree

Hide file tree

Showing 9 changed files with 95 additions and 28 deletions.
diff --git a/composer.json b/composer.json
@@ -49,7 +49,7 @@
         "composer/xdebug-handler": "^2.0",
         "infection/abstract-testframework-adapter": "^0.5.0",
         "infection/extension-installer": "^0.1.0",
-        "infection/include-interceptor": "^0.2.4",
+        "infection/include-interceptor": "^0.2.5",
         "justinrainbow/json-schema": "^5.2.10",
         "nikic/php-parser": "^4.10.3",
         "ocramius/package-versions": "^1.9.0 || ^2.0",

diff --git a/composer.lock b/composer.lock
diff --git a/src/Command/RunCommand.php b/src/Command/RunCommand.php
@@ -312,7 +312,7 @@ protected function configure(): void
                 self::OPTION_DEBUG,
                 null,
                 InputOption::VALUE_NONE,
-                'Will not clean up Infection temporary folder'
+                'Will not clean up utility files from Infection temporary folder. Adds command lines to the logs and prints Initial Tests output to stdout.'
             )
             ->addOption(
                 self::OPTION_DRY_RUN,

diff --git a/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php b/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php
@@ -37,6 +37,7 @@
 
 use Infection\Event\MutationTestingWasFinished;
 use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Component\Finder\Finder;
 
 /**
  * @internal
@@ -54,6 +55,11 @@ public function __construct(Filesystem $filesystem, string $tmpDir)
 
     public function onMutationTestingWasFinished(MutationTestingWasFinished $event): void
     {
-        $this->filesystem->remove($this->tmpDir);
+        $finder = Finder::create()
+            ->in($this->tmpDir)
+            // leave PHPUnit's result cache files so that subsequent Infection runs are faster because of `executionOrder=defects`
+            ->notName('/\.phpunit\.result\.cache\.(.*)/');
+
+        $this->filesystem->remove($finder);
     }
 }
diff --git a/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php b/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php
@@ -99,12 +99,10 @@ public function build(
             $originalBootstrapFile = $this->originalBootstrapFile = $this->getOriginalBootstrapFilePath($xPath);
         }
 
-        // if original phpunit.xml has order by random, we should replace it to use `default` order and our sorting
-        // by <file> tags (e.g. the fastest tests first)
-        $this->configManipulator->setDefaultTestsOrderAttribute($version, $xPath);
+        // activate PHPUnit's result cache and order tests by running defects first, then sorted by fastest first
+        $this->configManipulator->handleResultCacheAndExecutionOrder($version, $xPath, $mutationHash);
         $this->configManipulator->setStopOnFailure($xPath);
         $this->configManipulator->deactivateColours($xPath);
-        $this->configManipulator->deactivateResultCaching($xPath);
         $this->configManipulator->deactivateStderrRedirection($xPath);
         $this->configManipulator->removeExistingLoggers($xPath);
         $this->configManipulator->removeExistingPrinters($xPath);

diff --git a/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php b/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php
@@ -100,13 +100,21 @@ public function deactivateResultCaching(SafeDOMXPath $xPath): void
         $this->setAttributeValue($xPath, 'cacheResult', 'false');
     }
 
-    public function setDefaultTestsOrderAttribute(string $version, SafeDOMXPath $xPath): void
+    public function handleResultCacheAndExecutionOrder(string $version, SafeDOMXPath $xPath, string $mutationHash): void
     {
-        if (version_compare($version, '7.2', '<')) {
+        // starting from PHPUnit 7.3 we can set cache result and "defects" execution order https://github.com/sebastianbergmann/phpunit/blob/7.3.0/phpunit.xsd
+        if (version_compare($version, '7.3', '>=')) {
+            $this->setAttributeValue($xPath, 'cacheResult', 'true');
+            $this->setAttributeValue($xPath, 'cacheResultFile', sprintf('.phpunit.result.cache.%s', $mutationHash));
+            $this->setAttributeValue($xPath, 'executionOrder', 'defects');
+
             return;
         }
 
-        $this->setAttributeValue($xPath, 'executionOrder', 'default');
+        // from 7.2 to 7.3 we only can set "default" execution order and no cache result https://github.com/sebastianbergmann/phpunit/blob/7.2.0/phpunit.xsd
+        if (version_compare($version, '7.2', '>=')) {
+            $this->setAttributeValue($xPath, 'executionOrder', 'default');
+        }
     }
 
     public function deactivateStderrRedirection(SafeDOMXPath $xPath): void

diff --git a/tests/e2e/PestTestFramework/composer.json b/tests/e2e/PestTestFramework/composer.json
@@ -1,6 +1,6 @@
 {
     "require-dev": {
-        "pestphp/pest": "^1.2.0"
+        "pestphp/pest": "^1.15.0"
     },
     "autoload": {
         "psr-4": {

diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php
@@ -135,7 +135,7 @@ public function test_it_preserves_white_spaces_and_formatting(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.a1b2c3.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.a1b2c3.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests"/>
   </testsuites>
@@ -171,7 +171,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash1.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash1.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests">
       <file>/path/to/FooTest.php</file>
@@ -241,7 +241,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
   ~
   ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License
   -->
-<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash2.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" cacheResult="false" stderr="false">
+<phpunit backupGlobals="false" backupStaticAttributes="false" bootstrap="$tmp/interceptor.autoload.hash2.infection.php" colors="false" convertErrorsToExceptions="true" convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" syntaxCheck="false" stopOnFailure="true" stderr="false">
   <testsuites>
     <testsuite name="Infection testsuite with filtered tests">
       <file>/path/to/BarTest.php</file>
@@ -465,7 +465,26 @@ public function test_it_removes_printer_class(): void
         $this->assertSame(0, $printerClass->length);
     }
 
-    public function test_it_sets_default_execution_order_when_attribute_is_absent(): void
+    public function test_it_does_not_set_default_execution_order_for_phpunit_7_1(): void
+    {
+        $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml');
+
+        $xml = file_get_contents(
+            $builder->build(
+                [],
+                self::MUTATED_FILE_PATH,
+                self::HASH,
+                self::ORIGINAL_FILE_PATH,
+                '7.1'
+            )
+        );
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder');
+
+        $this->assertSame(0, $executionOrder->length);
+    }
+
+    public function test_it_sets_default_execution_order_when_attribute_is_absent_for_phpunit_7_2(): void
     {
         $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml');
 
@@ -484,7 +503,7 @@ public function test_it_sets_default_execution_order_when_attribute_is_absent():
         $this->assertSame('default', $executionOrder);
     }
 
-    public function test_it_sets_default_execution_order_when_attribute_is_present(): void
+    public function test_it_sets_default_execution_order_when_attribute_is_present_for_phpunit_7_2(): void
     {
         $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml');
 
@@ -503,6 +522,30 @@ public function test_it_sets_default_execution_order_when_attribute_is_present()
         $this->assertSame('default', $executionOrder);
     }
 
+    public function test_it_sets_defects_execution_order_and_cache_result_when_attribute_is_present_for_phpunit_7_3(): void
+    {
+        $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml');
+
+        $xml = file_get_contents(
+            $builder->build(
+                [],
+                self::MUTATED_FILE_PATH,
+                self::HASH,
+                self::ORIGINAL_FILE_PATH,
+                '7.3'
+            )
+        );
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue;
+        $this->assertSame('defects', $executionOrder);
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResult')[0]->nodeValue;
+        $this->assertSame('true', $executionOrder);
+
+        $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResultFile')[0]->nodeValue;
+        $this->assertSame(sprintf('.phpunit.result.cache.%s', self::HASH), $executionOrder);
+    }
+
     /**
      * @dataProvider locationsProvider
      *

diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php
@@ -600,7 +600,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x
         );
     }
 
-    public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void
+    public function test_it_activates_result_cache_and_execution_order_defects_for_phpunit_7_3(): void
     {
         $this->assertItChangesXML(<<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
@@ -611,20 +611,22 @@ public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void
 XML
             ,
             static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
-                $configManipulator->setDefaultTestsOrderAttribute('7.2', $xPath);
+                $configManipulator->handleResultCacheAndExecutionOrder('7.3', $xPath, 'a1b2c3');
             },
             <<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
 <phpunit
-    executionOrder="default"
+    executionOrder="defects"
+    cacheResult="true"
+    cacheResultFile=".phpunit.result.cache.a1b2c3"
     syntaxCheck="false"
 >
 </phpunit>
 XML
         );
     }
 
-    public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(): void
+    public function test_it_does_not_set_result_cache_for_phpunit_7_1(): void
     {
         $this->assertItChangesXML(<<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>
@@ -635,7 +637,7 @@ public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(
 XML
             ,
             static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
-                $configManipulator->setDefaultTestsOrderAttribute('7.1', $xPath);
+                $configManipulator->handleResultCacheAndExecutionOrder('7.1', $xPath, 'a1b2c3');
             },
             <<<'XML'
 <?xml version="1.0" encoding="UTF-8"?>