From fd989ebaa1b81a5d3ab7c68d483b60e22967f32a Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Sun, 8 Aug 2021 00:52:03 +0300 Subject: [PATCH 1/8] Cache phpunit results and run defects first to speed up the Mutation Testing execution When Infection runs Mutation Analysis for the first time, we can save the information about which Mutant was killed by which test, and then on the subsequent Infection execution (second and more), we can use this cached knowledge and run those test first that killed particular Mutant in a previous call. This should speed up MT as killed Mutants will be killed from the first "shot". --- composer.lock | 2 +- src/Command/RunCommand.php | 2 +- ...AfterMutationTestingFinishedSubscriber.php | 8 ++- .../Config/Builder/MutationConfigBuilder.php | 6 +-- .../Config/XmlConfigurationManipulator.php | 14 +++-- .../Builder/MutationConfigBuilderTest.php | 53 +++++++++++++++++-- .../XmlConfigurationManipulatorTest.php | 12 +++-- 7 files changed, 77 insertions(+), 20 deletions(-) diff --git a/composer.lock b/composer.lock index 5fce70f41..baf51ed45 100644 --- a/composer.lock +++ b/composer.lock @@ -4688,5 +4688,5 @@ "platform-overrides": { "php": "7.4.7" }, - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.1.0" } diff --git a/src/Command/RunCommand.php b/src/Command/RunCommand.php index 3768b0282..0a0969527 100644 --- a/src/Command/RunCommand.php +++ b/src/Command/RunCommand.php @@ -312,7 +312,7 @@ protected function configure(): void self::OPTION_DEBUG, null, InputOption::VALUE_NONE, - 'Will not clean up Infection temporary folder' + 'Will not clean up utility files from Infection temporary folder. Adds command lines to the logs and prints Initial Tests output to stdout.' ) ->addOption( self::OPTION_DRY_RUN, diff --git a/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php b/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php index 90a889e91..2b8b82100 100644 --- a/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php +++ b/src/Event/Subscriber/CleanUpAfterMutationTestingFinishedSubscriber.php @@ -37,6 +37,7 @@ use Infection\Event\MutationTestingWasFinished; use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Finder\Finder; /** * @internal @@ -54,6 +55,11 @@ public function __construct(Filesystem $filesystem, string $tmpDir) public function onMutationTestingWasFinished(MutationTestingWasFinished $event): void { - $this->filesystem->remove($this->tmpDir); + $finder = Finder::create() + ->in($this->tmpDir) + // leave PHPUnit's result cache files so that subsequent Infection runs are faster because of `executionOrder=defects` + ->notName('/\.phpunit\.result\.cache\.(.*)/'); + + $this->filesystem->remove($finder); } } diff --git a/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php b/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php index 4f31c0673..342377c73 100644 --- a/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php +++ b/src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php @@ -99,12 +99,10 @@ public function build( $originalBootstrapFile = $this->originalBootstrapFile = $this->getOriginalBootstrapFilePath($xPath); } - // if original phpunit.xml has order by random, we should replace it to use `default` order and our sorting - // by tags (e.g. the fastest tests first) - $this->configManipulator->setDefaultTestsOrderAttribute($version, $xPath); + // activate PHPUnit's result cache and order tests by running defects first, then sorted by fastest first + $this->configManipulator->handleResultCacheAndExecutionOrder($version, $xPath, $mutationHash); $this->configManipulator->setStopOnFailure($xPath); $this->configManipulator->deactivateColours($xPath); - $this->configManipulator->deactivateResultCaching($xPath); $this->configManipulator->deactivateStderrRedirection($xPath); $this->configManipulator->removeExistingLoggers($xPath); $this->configManipulator->removeExistingPrinters($xPath); diff --git a/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php b/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php index f29d19fb7..fe2397371 100644 --- a/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php +++ b/src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php @@ -100,13 +100,21 @@ public function deactivateResultCaching(SafeDOMXPath $xPath): void $this->setAttributeValue($xPath, 'cacheResult', 'false'); } - public function setDefaultTestsOrderAttribute(string $version, SafeDOMXPath $xPath): void + public function handleResultCacheAndExecutionOrder(string $version, SafeDOMXPath $xPath, string $mutationHash): void { - if (version_compare($version, '7.2', '<')) { + // starting from PHPUnit 7.3 we can set cache result and "defects" execution order https://github.com/sebastianbergmann/phpunit/blob/7.3.0/phpunit.xsd + if (version_compare($version, '7.3', '>=')) { + $this->setAttributeValue($xPath, 'cacheResult', 'true'); + $this->setAttributeValue($xPath, 'cacheResultFile', sprintf('.phpunit.result.cache.%s', $mutationHash)); + $this->setAttributeValue($xPath, 'executionOrder', 'defects'); + return; } - $this->setAttributeValue($xPath, 'executionOrder', 'default'); + // from 7.2 to 7.3 we only can set "default" execution order and no cache result https://github.com/sebastianbergmann/phpunit/blob/7.2.0/phpunit.xsd + if (version_compare($version, '7.2', '>=')) { + $this->setAttributeValue($xPath, 'executionOrder', 'default'); + } } public function deactivateStderrRedirection(SafeDOMXPath $xPath): void diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php index 1d8440d9d..702a648b3 100644 --- a/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php +++ b/tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php @@ -135,7 +135,7 @@ public function test_it_preserves_white_spaces_and_formatting(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + @@ -171,7 +171,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + /path/to/FooTest.php @@ -241,7 +241,7 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void ~ ~ License: https://opensource.org/licenses/BSD-3-Clause New BSD License --> - + /path/to/BarTest.php @@ -465,7 +465,26 @@ public function test_it_removes_printer_class(): void $this->assertSame(0, $printerClass->length); } - public function test_it_sets_default_execution_order_when_attribute_is_absent(): void + public function test_it_does_not_set_default_execution_order_for_phpunit_7_1(): void + { + $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml'); + + $xml = file_get_contents( + $builder->build( + [], + self::MUTATED_FILE_PATH, + self::HASH, + self::ORIGINAL_FILE_PATH, + '7.1' + ) + ); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder'); + + $this->assertSame(0, $executionOrder->length); + } + + public function test_it_sets_default_execution_order_when_attribute_is_absent_for_phpunit_7_2(): void { $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml'); @@ -484,7 +503,7 @@ public function test_it_sets_default_execution_order_when_attribute_is_absent(): $this->assertSame('default', $executionOrder); } - public function test_it_sets_default_execution_order_when_attribute_is_present(): void + public function test_it_sets_default_execution_order_when_attribute_is_present_for_phpunit_7_2(): void { $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml'); @@ -503,6 +522,30 @@ public function test_it_sets_default_execution_order_when_attribute_is_present() $this->assertSame('default', $executionOrder); } + public function test_it_sets_defects_execution_order_and_cache_result_when_attribute_is_present_for_phpunit_7_3(): void + { + $builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml'); + + $xml = file_get_contents( + $builder->build( + [], + self::MUTATED_FILE_PATH, + self::HASH, + self::ORIGINAL_FILE_PATH, + '7.3' + ) + ); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue; + $this->assertSame('defects', $executionOrder); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResult')[0]->nodeValue; + $this->assertSame('true', $executionOrder); + + $executionOrder = $this->queryXpath($xml, '/phpunit/@cacheResultFile')[0]->nodeValue; + $this->assertSame(sprintf('.phpunit.result.cache.%s', self::HASH), $executionOrder); + } + /** * @dataProvider locationsProvider * diff --git a/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php b/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php index 66e74d6aa..ec50970c1 100644 --- a/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php +++ b/tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php @@ -600,7 +600,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x ); } - public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void + public function test_it_activates_result_cache_and_execution_order_defects_for_phpunit_7_3(): void { $this->assertItChangesXML(<<<'XML' @@ -611,12 +611,14 @@ public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void XML , static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void { - $configManipulator->setDefaultTestsOrderAttribute('7.2', $xPath); + $configManipulator->handleResultCacheAndExecutionOrder('7.3', $xPath, 'a1b2c3'); }, <<<'XML' @@ -624,7 +626,7 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x ); } - public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(): void + public function test_it_does_not_set_result_cache_for_phpunit_7_1(): void { $this->assertItChangesXML(<<<'XML' @@ -635,7 +637,7 @@ public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1( XML , static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void { - $configManipulator->setDefaultTestsOrderAttribute('7.1', $xPath); + $configManipulator->handleResultCacheAndExecutionOrder('7.1', $xPath, 'a1b2c3'); }, <<<'XML' From da1f3bd89140487deba1ee4bbea0b4cb39e1bd87 Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 12:09:27 +0300 Subject: [PATCH 2/8] Debug why Pest e2e test fails --- tests/e2e/PestTestFramework/composer.json | 2 +- tests/e2e/PestTestFramework/infection.json | 2 +- tests/e2e/PestTestFramework/run_tests.bash | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/PestTestFramework/composer.json b/tests/e2e/PestTestFramework/composer.json index c6aa92a55..3db1f943d 100644 --- a/tests/e2e/PestTestFramework/composer.json +++ b/tests/e2e/PestTestFramework/composer.json @@ -1,6 +1,6 @@ { "require-dev": { - "pestphp/pest": "^1.2.0" + "pestphp/pest": "^1.15.0" }, "autoload": { "psr-4": { diff --git a/tests/e2e/PestTestFramework/infection.json b/tests/e2e/PestTestFramework/infection.json index 7029746fb..8586d10c8 100644 --- a/tests/e2e/PestTestFramework/infection.json +++ b/tests/e2e/PestTestFramework/infection.json @@ -6,7 +6,7 @@ ] }, "logs": { - "summary": "infection.log" + "text": "infection.log" }, "tmpDir": "." } diff --git a/tests/e2e/PestTestFramework/run_tests.bash b/tests/e2e/PestTestFramework/run_tests.bash index 949873a44..0aad1fe8c 100644 --- a/tests/e2e/PestTestFramework/run_tests.bash +++ b/tests/e2e/PestTestFramework/run_tests.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -readonly INFECTION="../../../bin/infection --test-framework=pest" +readonly INFECTION="../../../bin/infection --test-framework=pest --debug" set -e pipefail From 33cc6f266cb8884ef8e46fe949694f529b52d444 Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 12:27:37 +0300 Subject: [PATCH 3/8] Dump the log file for Pest e2e test --- tests/e2e/PestTestFramework/run_tests.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/PestTestFramework/run_tests.bash b/tests/e2e/PestTestFramework/run_tests.bash index 0aad1fe8c..5e89e034b 100644 --- a/tests/e2e/PestTestFramework/run_tests.bash +++ b/tests/e2e/PestTestFramework/run_tests.bash @@ -11,4 +11,5 @@ else php $INFECTION fi +cat infection.log diff -w expected-output.txt infection.log From b3f8b245fa94c5334c79bf0fab26ba8810c1d3fb Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 12:33:46 +0300 Subject: [PATCH 4/8] Log killed mutants for Pest e2e test --- tests/e2e/PestTestFramework/run_tests.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/PestTestFramework/run_tests.bash b/tests/e2e/PestTestFramework/run_tests.bash index 5e89e034b..1e63a0d40 100644 --- a/tests/e2e/PestTestFramework/run_tests.bash +++ b/tests/e2e/PestTestFramework/run_tests.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -readonly INFECTION="../../../bin/infection --test-framework=pest --debug" +readonly INFECTION="../../../bin/infection --test-framework=pest --debug --log-verbosity=all" set -e pipefail From 06ba57686613ae8fa8008cb77636244cba3d1e21 Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 13:07:46 +0300 Subject: [PATCH 5/8] Update `infection/include-interceptor` --- composer.lock | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index baf51ed45..cd8d9ca7a 100644 --- a/composer.lock +++ b/composer.lock @@ -183,16 +183,16 @@ }, { "name": "infection/include-interceptor", - "version": "0.2.4", + "version": "0.2.5", "source": { "type": "git", "url": "https://github.com/infection/include-interceptor.git", - "reference": "e3cf9317a7fd554ab60a5587f028b16418cc4264" + "reference": "0cc76d95a79d9832d74e74492b0a30139904bdf7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/infection/include-interceptor/zipball/e3cf9317a7fd554ab60a5587f028b16418cc4264", - "reference": "e3cf9317a7fd554ab60a5587f028b16418cc4264", + "url": "https://api.github.com/repos/infection/include-interceptor/zipball/0cc76d95a79d9832d74e74492b0a30139904bdf7", + "reference": "0cc76d95a79d9832d74e74492b0a30139904bdf7", "shasum": "" }, "require-dev": { @@ -223,9 +223,19 @@ "description": "Stream Wrapper: Include Interceptor. Allows to replace included (autoloaded) file with another one.", "support": { "issues": "https://github.com/infection/include-interceptor/issues", - "source": "https://github.com/infection/include-interceptor/tree/0.2.4" + "source": "https://github.com/infection/include-interceptor/tree/0.2.5" }, - "time": "2020-08-07T22:40:37+00:00" + "funding": [ + { + "url": "https://github.com/infection", + "type": "github" + }, + { + "url": "https://opencollective.com/infection", + "type": "open_collective" + } + ], + "time": "2021-08-09T10:03:57+00:00" }, { "name": "justinrainbow/json-schema", @@ -4688,5 +4698,5 @@ "platform-overrides": { "php": "7.4.7" }, - "plugin-api-version": "2.1.0" + "plugin-api-version": "2.0.0" } From 84750383b704f52a6e726d4f10995747b8715bbe Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 13:08:36 +0300 Subject: [PATCH 6/8] Revert debug changes in Pest e2e test --- tests/e2e/PestTestFramework/infection.json | 2 +- tests/e2e/PestTestFramework/run_tests.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/PestTestFramework/infection.json b/tests/e2e/PestTestFramework/infection.json index 8586d10c8..7029746fb 100644 --- a/tests/e2e/PestTestFramework/infection.json +++ b/tests/e2e/PestTestFramework/infection.json @@ -6,7 +6,7 @@ ] }, "logs": { - "text": "infection.log" + "summary": "infection.log" }, "tmpDir": "." } diff --git a/tests/e2e/PestTestFramework/run_tests.bash b/tests/e2e/PestTestFramework/run_tests.bash index 1e63a0d40..12f10f794 100644 --- a/tests/e2e/PestTestFramework/run_tests.bash +++ b/tests/e2e/PestTestFramework/run_tests.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -readonly INFECTION="../../../bin/infection --test-framework=pest --debug --log-verbosity=all" +readonly INFECTION="../../../bin/infection --test-framework=pest" set -e pipefail From 5f61f180d5b15d09bf3d233237b25581b8f0d779 Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 13:14:32 +0300 Subject: [PATCH 7/8] Remove debug `cat` --- tests/e2e/PestTestFramework/run_tests.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/PestTestFramework/run_tests.bash b/tests/e2e/PestTestFramework/run_tests.bash index 12f10f794..949873a44 100644 --- a/tests/e2e/PestTestFramework/run_tests.bash +++ b/tests/e2e/PestTestFramework/run_tests.bash @@ -11,5 +11,4 @@ else php $INFECTION fi -cat infection.log diff -w expected-output.txt infection.log From ff52f406f9a5c2a7f76415f6b28f743059c5b97e Mon Sep 17 00:00:00 2001 From: maks-rafalko Date: Mon, 9 Aug 2021 13:16:13 +0300 Subject: [PATCH 8/8] Set the minimum version of `infection/include-interceptor` - `0.2.5` --- composer.json | 2 +- composer.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index b27459b20..f41815f88 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "composer/xdebug-handler": "^2.0", "infection/abstract-testframework-adapter": "^0.5.0", "infection/extension-installer": "^0.1.0", - "infection/include-interceptor": "^0.2.4", + "infection/include-interceptor": "^0.2.5", "justinrainbow/json-schema": "^5.2.10", "nikic/php-parser": "^4.10.3", "ocramius/package-versions": "^1.9.0 || ^2.0", diff --git a/composer.lock b/composer.lock index cd8d9ca7a..0386bcb4a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d9cfb23acce60abd6aa6cc78193cd142", + "content-hash": "5905bdd842f6ae14114a96418dc3f578", "packages": [ { "name": "composer/xdebug-handler",