From bba07d871d27f06413cbbe2f1c138aa5c4fd03b2 Mon Sep 17 00:00:00 2001 From: Maks Rafalko Date: Sat, 4 Sep 2021 12:15:12 +0300 Subject: [PATCH] [Performance] Cache phpunit results and run defects first to speed up the Mutation Testing execution (#1549) * Cache phpunit results and run defects first to speed up the Mutation Testing execution When Infection runs Mutation Analysis for the first time, we can save the information about which Mutant was killed by which test, and then on the subsequent Infection execution (second and more), we can use this cached knowledge and run those test first that killed particular Mutant in a previous call. This should speed up MT as killed Mutants will be killed from the first "shot". * Debug why Pest e2e test fails * Dump the log file for Pest e2e test * Log killed mutants for Pest e2e test * Update `infection/include-interceptor` * Revert debug changes in Pest e2e test * Remove debug `cat` * Set the minimum version of `infection/include-interceptor` - `0.2.5` --- composer.json | 2 +- composer.lock | 24 ++++++--- src/Command/RunCommand.php | 2 +- ...AfterMutationTestingFinishedSubscriber.php | 8 ++- .../Config/Builder/MutationConfigBuilder.php | 6 +-- .../Config/XmlConfigurationManipulator.php | 14 +++-- tests/e2e/PestTestFramework/composer.json | 2 +- .../Builder/MutationConfigBuilderTest.php | 53 +++++++++++++++++-- .../XmlConfigurationManipulatorTest.php | 12 +++-- 9 files changed, 95 insertions(+), 28 deletions(-) diff --git a/composer.json b/composer.json index b27459b20..f41815f88 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "composer/xdebug-handler": "^2.0", "infection/abstract-testframework-adapter": "^0.5.0", "infection/extension-installer": "^0.1.0", - "infection/include-interceptor": "^0.2.4", + "infection/include-interceptor": "^0.2.5", "justinrainbow/json-schema": "^5.2.10", "nikic/php-parser": "^4.10.3", "ocramius/package-versions": "^1.9.0 || ^2.0", diff --git a/composer.lock b/composer.lock index 5fce70f41..0386bcb4a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d9cfb23acce60abd6aa6cc78193cd142", + "content-hash": "5905bdd842f6ae14114a96418dc3f578", "packages": [ { "name": "composer/xdebug-handler", @@ -183,16 +183,16 @@ }, { "name": "infection/include-interceptor", - "version": "0.2.4", + "version": "0.2.5", "source": { "type": "git", "url": "https://github.com/infection/include-interceptor.git", - "reference": "e3cf9317a7fd554ab60a5587f028b16418cc4264" + "reference": "0cc76d95a79d9832d74e74492b0a30139904bdf7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/infection/include-interceptor/zipball/e3cf9317a7fd554ab60a5587f028b16418cc4264", - "reference": "e3cf9317a7fd554ab60a5587f028b16418cc4264", + "url": "https://api.github.com/repos/infection/include-interceptor/zipball/0cc76d95a79d9832d74e74492b0a30139904bdf7", + "reference": "0cc76d95a79d9832d74e74492b0a30139904bdf7", "shasum": "" }, "require-dev": { @@ -223,9 +223,19 @@ "description": "Stream Wrapper: Include Interceptor. Allows to replace included (autoloaded) file with another one.", "support": { "issues": "https://github.com/infection/include-interceptor/issues", - "source": "https://github.com/infection/include-interceptor/tree/0.2.4" + "source": "https://github.com/infection/include-interceptor/tree/0.2.5" }, - "time": "2020-08-07T22:40:37+00:00" + "funding": [ + { + "url": "https://github.com/infection", + "type": "github" + }, + { + "url": "https://opencollective.com/infection", + "type": "open_collective" + } + ], + "time": "2021-08-09T10:03:57+00:00" }, { "name": "justinrainbow/json-schema", diff --git a/src/Command/RunCommand.php b/src/Command/RunCommand.php index 3768b0282..0a0969527 100644 --- a/src/Command/RunCommand.php +++ b/src/Command/RunCommand.php @@ -312,7 +312,7 @@ protected function configure(): void self::OPTION_DEBUG, null, InputOption::VALUE_NONE, - 'Will not clean up Infection temporary folder' + 'Will not clean up utility files from Infection temporary folder. Adds command lines to the logs and prints Initial Tests output to stdout.' ) ->addOption( self::OPTION_DRY_RUN, diff --git a/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php b/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php index 90a889e91..2b8b82100 100644 --- a/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php +++ b/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php @@ -37,6 +37,7 @@ use Infection\Event\MutationTestingWasFinished; use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Finder\Finder; /** * @internal @@ -54,6 +55,11 @@ public function __construct(Filesystem $filesystem, string $tmpDir) public function onMutationTestingWasFinished(MutationTestingWasFinished $event): void { - $this->filesystem->remove($this->tmpDir); + $finder = Finder::create() + ->in($this->tmpDir) + // leave PHPUnit's result cache files so that subsequent Infection runs are faster because of `executionOrder=defects` + ->notName('/\.phpunit\.result\.cache\.(.*)/'); + + $this->filesystem->remove($finder); } } diff --git a/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php b/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php index 4f31c0673..342377c73 100644 --- a/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php +++ b/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php @@ -99,12 +99,10 @@ public function build( $originalBootstrapFile = $this->originalBootstrapFile = $this->getOriginalBootstrapFilePath($xPath); } - // if original phpunit.xml has order by random, we should replace it to use `default` order and our sorting - // by tags (e.g. the fastest tests first) - $this->configManipulator->setDefaultTestsOrderAttribute($version, $xPath); + // activate PHPUnit's result cache and order tests by running defects first, then sorted by fastest first + $this->configManipulator->handleResultCacheAndExecutionOrder($version, $xPath, $mutationHash); $this->configManipulator->setStopOnFailure($xPath); $this->configManipulator->deactivateColours($xPath); - $this->configManipulator->deactivateResultCaching($xPath); $this->configManipulator->deactivateStderrRedirection($xPath); $this->configManipulator->removeExistingLoggers($xPath); $this->configManipulator->removeExistingPrinters($xPath); diff --git a/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php b/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php index f29d19fb7..fe2397371 100644 --- a/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php +++ b/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php @@ -100,13 +100,21 @@ public function deactivateResultCaching(SafeDOMXPath $xPath): void $this->setAttributeValue($xPath, 'cacheResult', 'false'); } - public function setDefaultTestsOrderAttribute(string $version, SafeDOMXPath $xPath): void + public function handleResultCacheAndExecutionOrder(string $version, SafeDOMXPath $xPath, string $mutationHash): void { - if (version_compare($version, '7.2', '<')) { + // starting from PHPUnit 7.3 we can set cache result and "defects" execution order https://github.com/sebastianbergmann/phpunit/blob/7.3.0/phpunit.xsd + if (version_compare($version, '7.3', '>=')) { + $this->setAttributeValue($xPath, 'cacheResult', 'true'); + $this->setAttributeValue($xPath, 'cacheResultFile', sprintf('.phpunit.result.cache.%s', $mutationHash)); + $this->setAttributeValue($xPath, 'executionOrder', 'defects'); + return; } - $this->setAttributeValue($xPath, 'executionOrder', 'default'); + // from 7.2 to 7.3 we only can set "default" execution order and no cache result https://github.com/sebastianbergmann/phpunit/blob/7.2.0/phpunit.xsd + if (version_compare($version, '7.2', '>=')) { + $this->setAttributeValue($xPath, 'executionOrder', 'default'); + } } public function deactivateStderrRedirection(SafeDOMXPath $xPath): void diff --git a/tests/e2e/PestTestFramework/composer.json b/tests/e2e/PestTestFramework/composer.json index c6aa92a55..3db1f943d 100644 --- a/tests/e2e/PestTestFramework/composer.json +++ b/tests/e2e/PestTestFramework/composer.json @@ -1,6 +1,6 @@ { "require-dev": { - "pestphp/pest": "^1.2.0" + "pestphp/pest": "^1.15.0" }, "autoload": { "psr-4": { diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php index 1d8440d9d..702a648b3 100644 --- a/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php +++ b/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php @@ -135,7 +135,7 @@ public function test_it_preserves_white_spaces_and_formatting(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + @@ -171,7 +171,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + /path/to/FooTest.php @@ -241,7 +241,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + /path/to/BarTest.php @@ -465,7 +465,26 @@ public function test_it_removes_printer_class(): void $this->assertSame(0, $printerClass->length); } - public function test_it_sets_default_execution_order_when_attribute_is_absent(): void + public function test_it_does_not_set_default_execution_order_for_phpunit_7_1(): void + { + $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml'); + + $xml = file_get_contents( + $builder->build( + [], + self::MUTATED_FILE_PATH, + self::HASH, + self::ORIGINAL_FILE_PATH, + '7.1' + ) + ); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder'); + + $this->assertSame(0, $executionOrder->length); + } + + public function test_it_sets_default_execution_order_when_attribute_is_absent_for_phpunit_7_2(): void { $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml'); @@ -484,7 +503,7 @@ public function test_it_sets_default_execution_order_when_attribute_is_absent(): $this->assertSame('default', $executionOrder); } - public function test_it_sets_default_execution_order_when_attribute_is_present(): void + public function test_it_sets_default_execution_order_when_attribute_is_present_for_phpunit_7_2(): void { $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml'); @@ -503,6 +522,30 @@ public function test_it_sets_default_execution_order_when_attribute_is_present() $this->assertSame('default', $executionOrder); } + public function test_it_sets_defects_execution_order_and_cache_result_when_attribute_is_present_for_phpunit_7_3(): void + { + $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml'); + + $xml = file_get_contents( + $builder->build( + [], + self::MUTATED_FILE_PATH, + self::HASH, + self::ORIGINAL_FILE_PATH, + '7.3' + ) + ); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue; + $this->assertSame('defects', $executionOrder); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResult')[0]->nodeValue; + $this->assertSame('true', $executionOrder); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResultFile')[0]->nodeValue; + $this->assertSame(sprintf('.phpunit.result.cache.%s', self::HASH), $executionOrder); + } + /** * @dataProvider locationsProvider * diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php index 66e74d6aa..ec50970c1 100644 --- a/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php +++ b/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php @@ -600,7 +600,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x ); } - public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void + public function test_it_activates_result_cache_and_execution_order_defects_for_phpunit_7_3(): void { $this->assertItChangesXML(<<<'XML' @@ -611,12 +611,14 @@ public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void XML , static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void { - $configManipulator->setDefaultTestsOrderAttribute('7.2', $xPath); + $configManipulator->handleResultCacheAndExecutionOrder('7.3', $xPath, 'a1b2c3'); }, <<<'XML' @@ -624,7 +626,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x ); } - public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(): void + public function test_it_does_not_set_result_cache_for_phpunit_7_1(): void { $this->assertItChangesXML(<<<'XML' @@ -635,7 +637,7 @@ public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1( XML , static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void { - $configManipulator->setDefaultTestsOrderAttribute('7.1', $xPath); + $configManipulator->handleResultCacheAndExecutionOrder('7.1', $xPath, 'a1b2c3'); }, <<<'XML'