Skip to content

Commit

Permalink
Allow running multiple suites individually and by wildcards in multi-…
Browse files Browse the repository at this point in the history 
…application setup.

Fix: #6434
  • Loading branch information
@calvinalkan
calvinalkan committed May 3, 2022
1 parent eb9c76e commit 538ecd5
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 26 deletions.
121 changes: 99 additions & 22 deletions src/Codeception/Command/Run.php
View file
Expand Up @@ -3,8 +3,6 @@

use Codeception\Codecept;
use Codeception\Configuration;
use Codeception\Lib\GroupManager;
use Codeception\Util\PathResolver;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
Expand Down Expand Up @@ -330,7 +328,7 @@ public function execute(InputInterface $input, OutputInterface $output)

$suite = $input->getArgument('suite');
$test = $input->getArgument('test');

if ($this->options['group']) {
$this->output->writeln(sprintf("[Groups] <info>%s</info> ", implode(', ', $this->options['group'])));
}
Expand All @@ -348,6 +346,7 @@ public function execute(InputInterface $input, OutputInterface $output)
foreach ($config['include'] as $include) {
// Find if the suite begins with an include path
if (strpos($suite, $include) === 0) {

// Use include config
$config = Configuration::config($projectDir.$include);

Expand All @@ -360,7 +359,7 @@ public function execute(InputInterface $input, OutputInterface $output)
$testsPath = $include . DIRECTORY_SEPARATOR. $config['paths']['tests'];

try {
list(, $suite, $test) = $this->matchTestFromFilename($suite, $testsPath);
[, $suite, $test] = $this->matchTestFromFilename($suite, $testsPath);
$isIncludeTest = true;
} catch (\InvalidArgumentException $e) {
// Incorrect include match, continue trying to find one
Expand All @@ -369,7 +368,7 @@ public function execute(InputInterface $input, OutputInterface $output)
} else {
$result = $this->matchSingleTest($suite, $config);
if ($result) {
list(, $suite, $test) = $result;
[, $suite, $test] = $result;
}
}
}
Expand All @@ -381,14 +380,18 @@ public function execute(InputInterface $input, OutputInterface $output)
} elseif (!empty($suite)) {
$result = $this->matchSingleTest($suite, $config);
if ($result) {
list(, $suite, $test) = $result;
[, $suite, $test] = $result;
}
}
}

if ($test) {
$userOptions['filter'] = $this->matchFilteredTestName($test);
} elseif ($suite) {
} elseif (
$suite
&& ! $this->isWildcardSuiteName($suite)
&& ! $this->isSuiteInMultiApplication($suite)
) {
$userOptions['filter'] = $this->matchFilteredTestName($suite);
}
if (!$this->options['silent'] && $config['settings']['shuffle']) {
Expand All @@ -405,18 +408,60 @@ public function execute(InputInterface $input, OutputInterface $output)

// Run all tests of given suite or all suites
if (!$test) {
$suites = $suite ? explode(',', $suite) : Configuration::suites();
$this->executed = $this->runSuites($suites, $this->options['skip']);

if (!empty($config['include']) and !$suite) {
$raw_suites = $suite ? explode(',', $suite) : Configuration::suites();

/** @var string[] $main_app_suites */
$main_app_suites = [];

/** @var array<string,string> $app_specific_suites */
$app_specific_suites = [];

/** @var string[] $wildcard_suites */
$wildcard_suites = [];

foreach ($raw_suites as $raw_suite) {
if($this->isWildcardSuiteName($raw_suite)){
$wildcard_suites[] = explode('*::', $raw_suite)[1];
continue;
}
if($this->isSuiteInMultiApplication($raw_suite)){
$app_and_suite = explode('::', $raw_suite);
$app_specific_suites[$app_and_suite[0]][] = $app_and_suite[1];
continue;
}
$main_app_suites[] = $raw_suite;
}

if([] !== $main_app_suites) {
$this->executed = $this->runSuites($main_app_suites, $this->options['skip']);
}

if(!empty($wildcard_suites) && ! empty($app_specific_suites)) {
$this->output->writeLn('<error>Wildcard options can not be combined with specific suites of included apps.</error>');
return self::INVALID;
}

if(!empty($config['include'])) {

$current_dir = Configuration::projectDir();
$suites += $config['include'];
$this->runIncludedSuites($config['include'], $current_dir);
$included_apps = $config['include'];

if(!empty($app_specific_suites)){
$included_apps = array_intersect($included_apps, array_keys($app_specific_suites));
}

$this->runIncludedSuites(
$included_apps,
$current_dir,
$app_specific_suites,
$wildcard_suites
);

}

if ($this->executed === 0) {
throw new \RuntimeException(
sprintf("Suite '%s' could not be found", implode(', ', $suites))
sprintf("Suite '%s' could not be found", implode(', ', $raw_suites))
);
}
}
Expand Down Expand Up @@ -469,7 +514,7 @@ protected function matchSingleTest($suite, $config)
if (strpos($realTestDir, $cwd) === 0) {
$file = $suite;
if (strpos($file, ':') !== false) {
list($file) = explode(':', $suite, -1);
[$file] = explode(':', $suite, -1);
}
$realPath = $cwd . DIRECTORY_SEPARATOR . $file;
if (file_exists($realPath) && strpos($realPath, $realTestDir) === 0) {
Expand All @@ -488,8 +533,10 @@ protected function matchSingleTest($suite, $config)
*
* @param array $suites
* @param string $parent_dir
* @param array<string,string[]> $filter_app_suites An array keyed by included app name where values are suite names to run.
* @param string[] $filter_suites_by_wildcard A list of suite names (applies to all included apps)
*/
protected function runIncludedSuites($suites, $parent_dir)
protected function runIncludedSuites($suites, $parent_dir, $filter_app_suites = [], $filter_suites_by_wildcard = [])
{
$defaultConfig = Configuration::config();
$absolutePath = \Codeception\Configuration::projectDir();
Expand All @@ -506,7 +553,15 @@ protected function runIncludedSuites($suites, $parent_dir)
}

$suites = Configuration::suites();


if(!empty($filter_suites_by_wildcard)){
$suites = array_intersect($suites, $filter_suites_by_wildcard);
}

if(isset($filter_app_suites[$relativePath])) {
$suites = array_intersect($suites, $filter_app_suites[$relativePath]);
}

$namespace = $this->currentNamespace();
$this->output->writeln(
"\n<fg=white;bg=magenta>\n[$namespace]: tests from $current_dir\n</fg=white;bg=magenta>"
Expand Down Expand Up @@ -534,6 +589,7 @@ protected function currentNamespace()

protected function runSuites($suites, $skippedSuites = [])
{

$executed = 0;
foreach ($suites as $suite) {
if (in_array($suite, $skippedSuites)) {
Expand All @@ -555,10 +611,10 @@ protected function matchTestFromFilename($filename, $testsPath)
if (strpos($filename, ':') !== false) {
if ((PHP_OS === 'Windows' || PHP_OS === 'WINNT') && $filename[1] === ':') {
// match C:\...
list($drive, $path, $filter) = explode(':', $filename, 3);
[$drive, $path, $filter] = explode(':', $filename, 3);
$filename = $drive . ':' . $path;
} else {
list($filename, $filter) = explode(':', $filename, 2);
[$filename, $filter] = explode(':', $filename, 2);
}

if ($filter) {
Expand Down Expand Up @@ -592,7 +648,7 @@ private function matchFilteredTestName(&$path)
{
$test_parts = explode(':', $path, 2);
if (count($test_parts) > 1) {
list($path, $filter) = $test_parts;
[$path, $filter] = $test_parts;
// use carat to signify start of string like in normal regex
// phpunit --filter matches against the fully qualified method name, so tests actually begin with :
$carat_pos = strpos($filter, '^');
Expand Down Expand Up @@ -661,4 +717,25 @@ private function ensurePhpExtIsAvailable($ext)
);
}
}

/**
* @param string $suite_name
*
* @return bool
*/
private function isWildcardSuiteName($suite_name)
{
return '*::' === substr($suite_name, 0, 3);
}

/**
* @param string $suite_name
*
* @return bool
*/
private function isSuiteInMultiApplication($suite_name)
{
return false !== strpos($suite_name, '::');
}

}
74 changes: 74 additions & 0 deletions tests/cli/IncludedCest.php
View file
Expand Up @@ -205,4 +205,78 @@ public function includedSuitesAreNotRunTwice (CliGuy $I) {
$I->seeInShellOutput('2 tests');
$I->dontSeeInShellOutput('4 tests');
}

/**
* @before moveToIncluded
* @param CliGuy $I
*/
public function someSuitesForSomeIncludedApplicationCanBeRun(CliGuy $I)
{
$I->executeCommand('run jazz::functional');

$I->seeInShellOutput('Jazz.functional Tests');
$I->dontSeeInShellOutput('Jazz.unit Tests');
$I->dontSeeInShellOutput('Shire.functional');

$I->executeCommand('run jazz::functional,jazz::unit');

$I->seeInShellOutput('Jazz.functional Tests');
$I->seeInShellOutput('Jazz.unit Tests');

$I->dontSeeInShellOutput('Shire.functional');

$I->executeCommand('run jazz::unit,shire::functional');

$I->seeInShellOutput('Shire.functional Tests');
$I->seeInShellOutput('Jazz.unit Tests');
$I->dontSeeInShellOutput('Jazz.functional Tests');

$I->executeCommand('run jazz/pianist::functional');

$I->dontSeeInShellOutput('Jazz.functional Tests');
$I->seeInShellOutput('Jazz\Pianist.functional');
}

/**
* @before moveToIncluded
* @param CliGuy $I
*/
public function someSuitesCanBeRunForAllIncludedApplications(\CliGuy $I)
{
$I->executeCommand('run *::functional');

// only functional tests are run
$I->seeInShellOutput('Jazz.functional Tests');
$I->seeInShellOutput('Jazz\Pianist.functional');
$I->seeInShellOutput('Shire.functional Tests');
// unit suites are not run
$I->dontSeeInShellOutput('Jazz.unit Tests');


$I->executeCommand('run *::unit');
// only unit tests are run
$I->seeInShellOutput('Jazz.unit Tests');
$I->dontSeeInShellOutput('Jazz.functional Tests');
$I->dontSeeInShellOutput('Jazz\Pianist.functional');
$I->dontSeeInShellOutput('Shire.functional Tests');

$I->executeCommand('run *::functional,*::unit');
// Both suites are run now
$I->seeInShellOutput('Jazz.functional Tests');
$I->seeInShellOutput('Jazz\Pianist.functional');
$I->seeInShellOutput('Shire.functional Tests');
$I->seeInShellOutput('Jazz.unit Tests');
}

/**
* @before moveToIncluded
* @param CliGuy $I
*/
public function wildCardSuitesAndAppSpecificSuitesCantBeCombined(CliGuy $I)
{
$I->executeCommand('run jazz::unit,*::functional', false);
$I->seeResultCodeIs(2);
$I->seeInShellOutput('Wildcard options can not be combined with specific suites of included apps.');
}

}
8 changes: 4 additions & 4 deletions tests/data/snapshots/tests/_data/Snapshot.NotAJsonSnapshot.xml
View file
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"></xs:schema>
<note>
<to>Son</to>
<from>Vader</from>
<heading>Disclaimer</heading>
<body>I'm your father!</body>
<to>Tove</to>
<from>Jani</from>
<heading>Reminder</heading>
<body>Don't forget me this weekend!</body>
</note>

0 comments on commit 538ecd5

Please sign in to comment.