diff --git a/src/Codeception/Events.php b/src/Codeception/Events.php index 6769200f66..285347ffbc 100644 --- a/src/Codeception/Events.php +++ b/src/Codeception/Events.php @@ -99,6 +99,14 @@ private function __construct() */ const TEST_WARNING = 'test.warning'; + /** + * The TEST_USELESS event occurs whenever test does not execute any assertions + * or when it calls expectNotToPerformAssertions and then performs some assertion. + * + * The event listener method receives a {@link \Codeception\Event\FailEvent} instance. + */ + const TEST_USELESS = 'test.useless'; + /** * The event listener method receives a {@link Codeception\Event\TestEvent} instance. diff --git a/src/Codeception/Subscriber/Console.php b/src/Codeception/Subscriber/Console.php index 2f07156040..6ef3f718a9 100644 --- a/src/Codeception/Subscriber/Console.php +++ b/src/Codeception/Subscriber/Console.php @@ -42,6 +42,7 @@ class Console implements EventSubscriberInterface Events::TEST_INCOMPLETE => 'testIncomplete', Events::TEST_SKIPPED => 'testSkipped', Events::TEST_WARNING => 'testWarning', + Events::TEST_USELESS => 'testUseless', Events::TEST_FAIL_PRINT => 'printFail', Events::RESULT_PRINT_AFTER => 'afterResult', ]; @@ -292,6 +293,11 @@ public function testIncomplete(FailEvent $e) $this->writelnFinishedTest($e, $this->message('I')->style('pending')); } + public function testUseless(FailEvent $event) + { + $this->writelnFinishedTest($event, $this->message('U')->style('pending')); + } + protected function isDetailed($test) { return $test instanceof ScenarioDriven && $this->steps; diff --git a/tests/cli/IncludedCest.php b/tests/cli/IncludedCest.php index 226df2d313..b74b4f9c87 100644 --- a/tests/cli/IncludedCest.php +++ b/tests/cli/IncludedCest.php @@ -212,7 +212,7 @@ public function includedSuitesAreNotRunTwice (CliGuy $I) { */ public function overwrittenConfigurationIsAlsoAppliedWhenRunningAnIncludedAppFromTheRootApp(CliGuy $I) { - if(PHP_VERSION_ID < 70100) { + if (version_compare(\PHPUnit\Runner\Version::id(), '7.0.0', '<')) { $I->markTestSkipped('This test can only run on PHP >= 7.1 because of reporter type declarations.'); } $I->executeCommand('run jazz/tests/functional --report -o "reporters: report: Jazz\CustomReporter1"'); diff --git a/tests/cli/RunUselessTestsCest.php b/tests/cli/RunUselessTestsCest.php new file mode 100644 index 0000000000..cccdd35a43 --- /dev/null +++ b/tests/cli/RunUselessTestsCest.php @@ -0,0 +1,150 @@ +amInPath('tests/data/useless'); + $I->executeCommand('run -v'); + // $I->seeInShellOutput('U UselessCept: Make no assertions'); + // $I->seeInShellOutput('U UselessCest: Make no assertions'); + $I->seeInShellOutput('U UselessTest: Make no assertions'); + $I->seeInShellOutput('OK, but incomplete, skipped, or risky tests!'); + + if (version_compare(\PHPUnit\Runner\Version::id(), '7.2.0', '>=')) { + $I->seeInShellOutput('There were 2 risky tests:'); + $I->seeInShellOutput('U UselessTest: Make unexpected assertion'); + } else { + $I->seeInShellOutput('There was 1 risky test:'); + $I->seeInShellOutput('S UselessTest: Make unexpected assertion'); + } + + if (DIRECTORY_SEPARATOR === '/') { +// $I->seeInShellOutput( +// '1) UselessCept: Make no assertions +// Test tests/unit/UselessCept.php +//This test did not perform any assertions' +// ); +// $I->seeInShellOutput( +// ' +//2) UselessCest: Make no assertions +// Test tests/unit/UselessCest.php:makeNoAssertions +//This test did not perform any assertions +// +//Scenario Steps: +// +// 1. // make no assertions' +// ); + $I->seeInShellOutput( + ' +1) UselessTest: Make no assertions + Test tests/unit/UselessTest.php:testMakeNoAssertions +This test did not perform any assertions' + ); + if (version_compare(\PHPUnit\Runner\Version::id(), '7.2.0', '>=')) { + $I->seeInShellOutput( + ' +2) UselessTest: Make unexpected assertion + Test tests/unit/UselessTest.php:testMakeUnexpectedAssertion +This test is annotated with "@doesNotPerformAssertions" but performed 1 assertions' + ); + } + + return; + } + +// $I->seeInShellOutput( +// '1) UselessCept: Make no assertions +// Test tests\unit\UselessCept.php +//This test did not perform any assertions' +// ); +// $I->seeInShellOutput( +// ' +//2) UselessCest: Make no assertions +// Test tests\unit\UselessCest.php:makeNoAssertions +//This test did not perform any assertions +// +//Scenario Steps: +// +// 1. // make no assertions' +// ); + $I->seeInShellOutput( + ' +1) UselessTest: Make no assertions + Test tests\unit\UselessTest.php:testMakeNoAssertions +This test did not perform any assertions' + ); + if (version_compare(\PHPUnit\Runner\Version::id(), '7.2.0', '>=')) { + $I->seeInShellOutput( + ' +2) UselessTest: Make unexpected assertion + Test tests\unit\UselessTest.php:testMakeUnexpectedAssertion +This test is annotated with "@doesNotPerformAssertions" but performed 1 assertions' + ); + } + } + + public function checkReports(CliGuy $I) + { + $I->amInPath('tests/data/useless'); + $I->executeCommand('run --report --xml --phpunit-xml --html'); + + if (version_compare(\PHPUnit\Runner\Version::id(), '7.2.0', '>=')) { + $useless = 2; + $skipped = 0; + $assertions = 1; + $I->seeInShellOutput('UselessTest: Make unexpected assertion.....................................Useless'); + } else { + $useless = 1; + $skipped = 1; + $assertions = 0; + $I->seeInShellOutput('UselessTest: Make unexpected assertion.....................................Skipped'); + } + $I->seeInShellOutput("Skipped: $skipped. Useless: $useless"); + + // $I->seeInShellOutput('UselessCept: Make no assertions............................................Useless'); + // $I->seeInShellOutput('UselessCest: Make no assertions............................................Useless'); + $I->seeInShellOutput('UselessTest: Make no assertions............................................Useless'); + + $I->seeFileFound('report.xml', 'tests/_output'); + + if (version_compare(\PHPUnit\Runner\Version::id(), '6.0.0', '>=')) { + $I->seeInThisFile( + "seeInThisFile( + "seeInThisFile('seeInThisFile('seeInThisFile('seeInThisFile( + "seeInThisFile('seeInThisFile('seeInThisFile('Useless scenarios:'); + $I->seeInThisFile("$useless"); + $I->seeInThisFile('UselessCest'); + $I->seeInThisFile('UselessTest'); + $I->seeInThisFile('UselessCept'); + } +} diff --git a/tests/data/useless/codeception.yml b/tests/data/useless/codeception.yml new file mode 100644 index 0000000000..01574bdf2e --- /dev/null +++ b/tests/data/useless/codeception.yml @@ -0,0 +1,11 @@ +paths: + tests: tests + output: tests/_output + data: tests/_data + support: tests/_support + envs: tests/_envs +actor_suffix: Tester +settings: + log_incomplete_skipped: true + report_useless_tests: true + colors: false \ No newline at end of file diff --git a/tests/data/useless/tests/_output/.gitignore b/tests/data/useless/tests/_output/.gitignore new file mode 100644 index 0000000000..c96a04f008 --- /dev/null +++ b/tests/data/useless/tests/_output/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/tests/data/useless/tests/_support/UnitTester.php b/tests/data/useless/tests/_support/UnitTester.php new file mode 100644 index 0000000000..28353572c1 --- /dev/null +++ b/tests/data/useless/tests/_support/UnitTester.php @@ -0,0 +1,26 @@ +wantTo('make no assertions'); diff --git a/tests/data/useless/tests/unit/UselessCest.php b/tests/data/useless/tests/unit/UselessCest.php new file mode 100644 index 0000000000..c2c4aac422 --- /dev/null +++ b/tests/data/useless/tests/unit/UselessCest.php @@ -0,0 +1,9 @@ +comment('make no assertions'); + } +} diff --git a/tests/data/useless/tests/unit/UselessTest.php b/tests/data/useless/tests/unit/UselessTest.php new file mode 100644 index 0000000000..1c1b542bda --- /dev/null +++ b/tests/data/useless/tests/unit/UselessTest.php @@ -0,0 +1,23 @@ +markTestSkipped('Not supported before PHPUnit 7.2'); + } + $this->expectNotToPerformAssertions(); + $this->assertTrue(true); + } +}