Skip to content

Commit

Permalink
For Mutant's phpunit.xml, set executionOrder="default" to prevent r…
Browse files Browse the repository at this point in the history 
…andom ordering of the tests since we need them to be sorted (fastest - first) (#1547)

When Infection build Mutant's `phpunit.xml` file, we add `<file>` tags, sorted by tests execution time, so that the fastest tests runs first.

But when the original `phpunit.xml` contains `executionOrder="default"`, it was inherited to Mutant's `phpunit.xml` and broke the order, running the tests in a random order each time `infection` was executed.

This also explains why I got different execution time of Infection for the project with *functional* tests.

When Mutant is covered by 1000 functional tests, and tests are executed by random, it can be killed by the first test in run #1 and can be killed by 1000-th test in run #2, if the killing test is executed the last.

With this update, all the tests will be executed in the same orders (fastest - first) in all the infection runs.
  • Loading branch information
@maks-rafalko
maks-rafalko committed Aug 7, 2021
1 parent 01f1f62 commit 5c6b61f
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 18 deletions.
3 changes: 2 additions & 1 deletion src/TestFramework/AbstractTestFrameworkAdapter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,8 @@ protected function buildMutationConfigFile(
$tests,
$mutantFilePath,
$mutationHash,
$mutationOriginalFilePath
$mutationOriginalFilePath,
$this->getVersion()
);
}

Expand Down
3 changes: 2 additions & 1 deletion src/TestFramework/Config/MutationConfigBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ abstract public function build(
array $tests,
string $mutantFilePath,
string $mutationHash,
string $mutationOriginalFilePath
string $mutationOriginalFilePath,
string $version
): string;

protected function getInterceptorFileContent(string $interceptorPath, string $originalFilePath, string $mutantFilePath): string
Expand Down
6 changes: 5 additions & 1 deletion src/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ public function build(
array $tests,
string $mutantFilePath,
string $mutationHash,
string $mutationOriginalFilePath
string $mutationOriginalFilePath,
string $version
): string {
$dom = $this->getDom();
$xPath = new SafeDOMXPath($dom);
Expand All @@ -98,6 +99,9 @@ public function build(
$originalBootstrapFile = $this->originalBootstrapFile = $this->getOriginalBootstrapFilePath($xPath);
}

// if original phpunit.xml has order by random, we should replace it to use `default` order and our sorting
// by <file> tags (e.g. the fastest tests first)
$this->configManipulator->setDefaultTestsOrderAttribute($version, $xPath);
$this->configManipulator->setStopOnFailure($xPath);
$this->configManipulator->deactivateColours($xPath);
$this->configManipulator->deactivateResultCaching($xPath);
Expand Down
10 changes: 10 additions & 0 deletions src/TestFramework/PhpUnit/Config/XmlConfigurationManipulator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
use LibXMLError;
use LogicException;
use function Safe\sprintf;
use function version_compare;
use Webmozart\Assert\Assert;

/**
Expand Down Expand Up @@ -99,6 +100,15 @@ public function deactivateResultCaching(SafeDOMXPath $xPath): void
$this->setAttributeValue($xPath, 'cacheResult', 'false');
}

public function setDefaultTestsOrderAttribute(string $version, SafeDOMXPath $xPath): void
{
if (version_compare($version, '7.2', '<')) {
return;
}

$this->setAttributeValue($xPath, 'executionOrder', 'default');
}

public function deactivateStderrRedirection(SafeDOMXPath $xPath): void
{
$this->setAttributeValue($xPath, 'stderr', 'false');
Expand Down
83 changes: 68 additions & 15 deletions tests/phpunit/TestFramework/PhpUnit/Config/Builder/MutationConfigBuilderTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ public function test_it_builds_and_dump_the_xml_configuration(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
);

$this->assertSame(
Expand All @@ -119,7 +120,8 @@ public function test_it_preserves_white_spaces_and_formatting(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
);

$tmp = $this->tmp;
Expand Down Expand Up @@ -200,7 +202,8 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
],
self::MUTATED_FILE_PATH,
'hash1',
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
)
);
Expand Down Expand Up @@ -269,7 +272,8 @@ public function test_it_can_build_the_config_for_multiple_mutations(): void
],
self::MUTATED_FILE_PATH,
'hash2',
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
)
);
Expand Down Expand Up @@ -308,7 +312,8 @@ public function test_it_builds_path_to_mutation_config_file(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);
}
Expand All @@ -320,7 +325,8 @@ public function test_it_sets_custom_autoloader(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand Down Expand Up @@ -348,7 +354,8 @@ public function test_it_sets_custom_autoloader_when_attribute_is_absent(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -374,7 +381,8 @@ public function test_it_sets_stops_on_failure(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -390,7 +398,8 @@ public function test_it_deactivates_the_colors(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -407,7 +416,8 @@ public function test_it_handles_root_test_suite(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
);

$testSuite = $this->queryXpath(
Expand All @@ -426,7 +436,8 @@ public function test_it_removes_original_loggers(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -443,7 +454,8 @@ public function test_it_removes_printer_class(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -453,6 +465,44 @@ public function test_it_removes_printer_class(): void
$this->assertSame(0, $printerClass->length);
}

public function test_it_sets_default_execution_order_when_attribute_is_absent(): void
{
$builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_without_coverage_whitelist.xml');

$xml = file_get_contents(
$builder->build(
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH,
'7.2'
)
);

$executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue;

$this->assertSame('default', $executionOrder);
}

public function test_it_sets_default_execution_order_when_attribute_is_present(): void
{
$builder = $this->createConfigBuilder(self::FIXTURES . '/phpunit_with_order_set.xml');

$xml = file_get_contents(
$builder->build(
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH,
'7.2'
)
);

$executionOrder = $this->queryXpath($xml, '/phpunit/@executionOrder')[0]->nodeValue;

$this->assertSame('default', $executionOrder);
}

/**
* @dataProvider locationsProvider
*
Expand All @@ -468,7 +518,8 @@ public function test_it_sets_sorted_list_of_test_files(
$tests,
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -492,7 +543,8 @@ public function test_it_removes_default_test_suite(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
)
);

Expand All @@ -510,7 +562,8 @@ public function test_interceptor_is_included(): void
[],
self::MUTATED_FILE_PATH,
self::HASH,
self::ORIGINAL_FILE_PATH
self::ORIGINAL_FILE_PATH,
'7.1'
);

$expectedCustomAutoloadFilePath = sprintf(
Expand Down
47 changes: 47 additions & 0 deletions tests/phpunit/TestFramework/PhpUnit/Config/XmlConfigurationManipulatorTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,53 @@ static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $x
);
}

public function test_it_sets_execution_order_to_default_for_phpunit_7_2(): void
{
$this->assertItChangesXML(<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<phpunit
syntaxCheck="false"
>
</phpunit>
XML
,
static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
$configManipulator->setDefaultTestsOrderAttribute('7.2', $xPath);
},
<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<phpunit
executionOrder="default"
syntaxCheck="false"
>
</phpunit>
XML
);
}

public function test_it_does_not_set_execution_order_to_default_for_phpunit_7_1(): void
{
$this->assertItChangesXML(<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<phpunit
syntaxCheck="false"
>
</phpunit>
XML
,
static function (XmlConfigurationManipulator $configManipulator, SafeDOMXPath $xPath): void {
$configManipulator->setDefaultTestsOrderAttribute('7.1', $xPath);
},
<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<phpunit
syntaxCheck="false"
>
</phpunit>
XML
);
}

public function test_it_removes_existing_printers(): void
{
$this->assertItChangesXML(<<<'XML'
Expand Down

0 comments on commit 5c6b61f

Please sign in to comment.