Improve multi-application experience, allow including suites by name (opposite of --skip)

src/Codeception/Command/Run.php

@@ -3,8 +3,6 @@
 
 use Codeception\Codecept;
 use Codeception\Configuration;
-use Codeception\Lib\GroupManager;
-use Codeception\Util\PathResolver;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -392,7 +390,11 @@ public function execute(InputInterface $input, OutputInterface $output)
 
         if ($test) {
             $userOptions['filter'] = $this->matchFilteredTestName($test);
-        } elseif ($suite) {
+        } elseif (
+            $suite
+            && ! $this->isWildcardSuiteName($suite)
+            && ! $this->isSuiteInMultiApplication($suite)
+        ) {
             $userOptions['filter'] = $this->matchFilteredTestName($suite);
         }
         if (!$this->options['silent'] && $config['settings']['shuffle']) {
@@ -409,18 +411,66 @@ public function execute(InputInterface $input, OutputInterface $output)
 
         // Run all tests of given suite or all suites
         if (!$test) {
-            $suites = $suite ? explode(',', $suite) : Configuration::suites();
-            $this->executed = $this->runSuites($suites, $this->options['skip']);
-
-            if (!empty($config['include']) and !$suite) {
-                $current_dir = Configuration::projectDir();
-                $suites += $config['include'];
-                $this->runIncludedSuites($config['include'], $current_dir);
+
+            $didPassCliSuite = !empty($suite);
+
+            $rawSuites = $didPassCliSuite ? explode(',', $suite) : Configuration::suites();
+
+            /** @var string[] $mainAppSuites */
+            $mainAppSuites = [];
+
+            /** @var array<string,string> $appSpecificSuites */
+            $appSpecificSuites = [];
+
+            /** @var string[] $wildcardSuites */
+            $wildcardSuites = [];
+
+            foreach ($rawSuites as $rawSuite) {
+                if($this->isWildcardSuiteName($rawSuite)){
+                    $wildcardSuites[] = explode('*::', $rawSuite)[1];
+                    continue;
+                }
+                if($this->isSuiteInMultiApplication($rawSuite)){
+                    $appAndSuite = explode('::', $rawSuite);
+                    $appSpecificSuites[$appAndSuite[0]][] = $appAndSuite[1];
+                    continue;
+                }
+                $mainAppSuites[] = $rawSuite;
             }
-
+
+            if([] !== $mainAppSuites) {
+                $this->executed = $this->runSuites($mainAppSuites, $this->options['skip']);
+            }
+
+            if(!empty($wildcardSuites) && ! empty($appSpecificSuites)) {
+                $this->output->writeLn('<error>Wildcard options can not be combined with specific suites of included apps.</error>');
+                return 2;
+            }
+
+            if(
+                !empty($config['include'])
+                && (!$didPassCliSuite || !empty($wildcardSuites) || !empty($appSpecificSuites))
+            ) {
+
+                $currentDir = Configuration::projectDir();
+                $includedApps = $config['include'];
+
+                if(!empty($appSpecificSuites)){
+                    $includedApps = array_intersect($includedApps, array_keys($appSpecificSuites));
+                }
+
+                $this->runIncludedSuites(
+                    $includedApps,
+                    $currentDir,
+                    $appSpecificSuites,
+                    $wildcardSuites
+                );
+
+            }
+
             if ($this->executed === 0) {
                 throw new \RuntimeException(
-                    sprintf("Suite '%s' could not be found", implode(', ', $suites))
+                    sprintf("Suite '%s' could not be found", implode(', ', $rawSuites))
                 );
             }
         }
@@ -492,8 +542,10 @@ protected function matchSingleTest($suite, $config)
      *
      * @param array $suites
      * @param string $parent_dir
+     * @param array<string,string[]> $filterAppSuites An array keyed by included app name where values are suite names to run.
+     * @param string[] $filterSuitesByWildcard A list of suite names (applies to all included apps)
      */
-    protected function runIncludedSuites($suites, $parent_dir)
+    protected function runIncludedSuites($suites, $parent_dir, $filterAppSuites = [], $filterSuitesByWildcard = [])
     {
         $defaultConfig = Configuration::config();
         $absolutePath = \Codeception\Configuration::projectDir();
@@ -502,15 +554,23 @@ protected function runIncludedSuites($suites, $parent_dir)
             $current_dir = rtrim($parent_dir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $relativePath;
             $config = Configuration::config($current_dir);
 
-            if (!empty($defaultConfig['groups'])) {
+            if ( !empty($defaultConfig['groups'])) {
                 $groups = array_map(function($g) use ($absolutePath) {
                     return $absolutePath . $g;
                 }, $defaultConfig['groups']);
                 Configuration::append(['groups' => $groups]);
             }
 
             $suites = Configuration::suites();
-
+
+            if( !empty($filterSuitesByWildcard)){
+                $suites = array_intersect($suites, $filterSuitesByWildcard);
+            }
+
+            if( isset($filterAppSuites[$relativePath])) {
+                $suites = array_intersect($suites, $filterAppSuites[$relativePath]);
+            }
+
             $namespace = $this->currentNamespace();
             $this->output->writeln(
                 "\n<fg=white;bg=magenta>\n[$namespace]: tests from $current_dir\n</fg=white;bg=magenta>"
@@ -665,4 +725,34 @@ private function ensurePhpExtIsAvailable($ext)
             );
         }
     }
+
+    /**
+     * @param  string  $suite_name
+     *
+     * @return bool
+     */
+    private function isWildcardSuiteName($suite_name)
+    {
+        return '*::' === substr($suite_name, 0, 3);
+    }
+
+    /**
+     * @param  string  $suite_name
+     *
+     * @return bool
+     */
+    private function isSuiteInMultiApplication($suite_name)
+    {
+        return false !== strpos($suite_name, '::');
+    }
+
+    /**
+     * @param  string  $suite_name
+     *
+     * @return bool
+     */
+    private function isRootLevelSuite($suite_name) {
+        return !$this->isSuiteInMultiApplication($suite_name) && ! $this->isWildcardSuiteName($suite_name);
+    }
+
 }

tests/cli/IncludedCest.php

@@ -219,5 +219,144 @@ public function overwrittenConfigurationIsAlsoAppliedWhenRunningAnIncludedAppFro
         $I->seeInShellOutput('CustomReporter1');
     }
 
+    /**
+     * @before moveToIncluded
+     * @param CliGuy $I
+     */
+    public function someSuitesForSomeIncludedApplicationCanBeRun(CliGuy $I)
+    {
+        $I->executeCommand('run jazz::functional');
+
+        $I->seeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz.unit Tests');
+        $I->dontSeeInShellOutput('Shire.functional');
+
+        $I->executeCommand('run jazz::functional,jazz::unit');
+
+        $I->seeInShellOutput('Jazz.functional Tests');
+        $I->seeInShellOutput('Jazz.unit Tests');
+
+        $I->dontSeeInShellOutput('Shire.functional');
+
+        $I->executeCommand('run jazz::unit,shire::functional');
+
+        $I->seeInShellOutput('Shire.functional Tests');
+        $I->seeInShellOutput('Jazz.unit Tests');
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+
+        $I->executeCommand('run jazz/pianist::functional');
+
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+        $I->seeInShellOutput('Jazz\Pianist.functional');
+    }
+
+    /**
+     * @before moveToIncluded
+     * @param CliGuy $I
+     */
+    public function someSuitesCanBeRunForAllIncludedApplications(\CliGuy $I)
+    {
+        $I->executeCommand('run *::functional');
+
+        // only functional tests are run
+        $I->seeInShellOutput('Jazz.functional Tests');
+        $I->seeInShellOutput('Jazz\Pianist.functional');
+        $I->seeInShellOutput('Shire.functional Tests');
+        // unit suites are not run
+        $I->dontSeeInShellOutput('Jazz.unit Tests');
+
+
+        $I->executeCommand('run *::unit');
+        // only unit tests are run
+        $I->seeInShellOutput('Jazz.unit Tests');
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz\Pianist.functional');
+        $I->dontSeeInShellOutput('Shire.functional Tests');
+
+        $I->executeCommand('run *::functional,*::unit');
+        // Both suites are run now
+        $I->seeInShellOutput('Jazz.functional Tests');
+        $I->seeInShellOutput('Jazz\Pianist.functional');
+        $I->seeInShellOutput('Shire.functional Tests');
+        $I->seeInShellOutput('Jazz.unit Tests');
+    }
+
+    /**
+     * @before moveToIncluded
+     * @param CliGuy $I
+     */
+    public function wildCardSuitesAndAppSpecificSuitesCantBeCombined(CliGuy $I)
+    {
+        $I->executeCommand('run jazz::unit,*::functional', false);
+        $I->seeResultCodeIs(2);
+        $I->seeInShellOutput('Wildcard options can not be combined with specific suites of included apps.');
+    }
+
+    /**
+     * @before moveToIncluded
+     * @param CliGuy $I
+     */
+    public function runningASuiteInTheRootApplicationDoesNotRunTheIncludedAppSuites(CliGuy $I)
+    {
+        $I->executeCommand('run unit');
+
+        $I->seeInShellOutput('Unit Tests (1)');
+        $I->seeInShellOutput('RootApplicationUnitTest:');
+
+        $I->dontSeeInShellOutput('Functional Tests (1)');
+        $I->dontSeeInShellOutput('RootApplicationFunctionalTest:');
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz.unit Tests');
+
+        $I->executeCommand('run functional');
+
+        $I->seeInShellOutput('Functional Tests (1)');
+        $I->seeInShellOutput('RootApplicationFunctionalTest:');
+
+        $I->dontSeeInShellOutput('Unit Tests (1)');
+        $I->dontSeeInShellOutput('RootApplicationUnitTest:');
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz.unit Tests');
+    }
+
+    /**
+     * @before moveToIncluded
+     * @param CliGuy $I
+     */
+    public function rootSuitesCanBeRunInCombinationWithIncludedSuites(CliGuy $I)
+    {
+        $I->executeCommand('run unit,*::unit');
+
+        // root level
+        $I->seeInShellOutput('Unit Tests (1)');
+        $I->seeInShellOutput('RootApplicationUnitTest:');
+        $I->dontSeeInShellOutput('Functional Tests (1)');
+        $I->dontSeeInShellOutput('RootApplicationFunctionalTest:');
+
+        // included
+        $I->seeInShellOutput('Jazz.unit Tests');
+        $I->dontSeeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz\Pianist.functional');
+        $I->dontSeeInShellOutput('Shire.functional Tests');
+
+        // Ensure that root level suites are not run twice.
+        $I->seeInShellOutput('OK (3 tests, 3 assertions)');
+
+
+        $I->executeCommand('run unit,jazz::functional');
+
+        // root level
+        $I->seeInShellOutput('Unit Tests (1)');
+        $I->seeInShellOutput('RootApplicationUnitTest:');
+        $I->dontSeeInShellOutput('Functional Tests (1)');
+        $I->dontSeeInShellOutput('RootApplicationFunctionalTest:');
+
+        // included apps
+        $I->seeInShellOutput('Jazz.functional Tests');
+        $I->dontSeeInShellOutput('Jazz.unit Tests');
+        $I->dontSeeInShellOutput('Jazz\Pianist.functional');
+        $I->dontSeeInShellOutput('Shire.functional Tests');
+
+    }
 
 }

tests/data/included/codeception.yml

@@ -3,10 +3,15 @@ include:
   - jazz/pianist
   - shire
 
+
+
 groups:
   group: group.txt
 
 paths:
+  tests: tests
+  data: tests/_data
+  support: tests/_support
   log: "_log"
 settings:
   colors: false

tests/data/included/tests/functional.suite.yml

@@ -0,0 +1 @@
+##

tests/data/included/tests/functional/RootApplicationFunctionalTest.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace data\included\tests\functional;
+
+use Codeception\Test\Unit;
+
+/**
+ * This is not a Cept test case because it does not matter for what we are testing.
+ */
+class RootApplicationFunctionalTest extends Unit
+{
+
+    public function testBarEqualsBar()
+    {
+        $this->assertSame('bar', 'bar');
+    }
+}

tests/data/included/tests/unit.suite.yml

@@ -0,0 +1 @@
+##

tests/data/included/tests/unit/RootApplicationUnitTest.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace data\included\tests\unit;
+
+use Codeception\Test\Unit;
+
+class RootApplicationUnitTest extends Unit
+{
+
+    public function testFooEqualsFoo()
+    {
+        $this->assertSame('foo', 'foo');
+    }
+
+}