Fail Infection execution when at least 1 ignore source code regex was…

…n't matched

Implements #1604

Will be useful to remove stale/non-matching `ignoreSourceCodeByRegex` settings inside `infection.json` and will help debugging them.

src/Command/RunCommand.php

@@ -56,6 +56,7 @@
 use Infection\Logger\GitHub\NoFilesInDiffToMutate;
 use Infection\Metrics\MinMsiCheckFailed;
 use Infection\Process\Runner\InitialTestsFailed;
+use Infection\Process\Runner\NotMatchedIgnoreSourceCodeRegexFound;
 use Infection\TestFramework\TestFrameworkTypes;
 use InvalidArgumentException;
 use const PHP_SAPI;
@@ -355,7 +356,7 @@ protected function executeCommand(IO $io): bool
             $io->success($e->getMessage());
 
             return true;
-        } catch (InitialTestsFailed | MinMsiCheckFailed $exception) {
+        } catch (InitialTestsFailed | MinMsiCheckFailed | NotMatchedIgnoreSourceCodeRegexFound $exception) {
             // TODO: we can move that in a dedicated logger later and handle those cases in the
             // Engine instead
             $io->error($exception->getMessage());

src/Process/Runner/MutationTestingRunner.php

@@ -35,6 +35,10 @@
 
 namespace Infection\Process\Runner;
 
+use function array_keys;
+use function array_merge;
+use function array_unique;
+use function array_values;
 use Infection\Differ\DiffSourceCodeMatcher;
 use Infection\Event\EventDispatcher\EventDispatcher;
 use Infection\Event\MutantProcessWasFinished;
@@ -47,6 +51,7 @@
 use Infection\Mutation\Mutation;
 use Infection\Process\Factory\MutantProcessFactory;
 use function Pipeline\take;
+use function Safe\array_flip;
 use Symfony\Component\Filesystem\Filesystem;
 
 /**
@@ -99,15 +104,19 @@ public function run(iterable $mutations, string $testFrameworkExtraOptions): voi
         $numberOfMutants = IterableCounter::bufferAndCountIfNeeded($mutations, $this->runConcurrently);
         $this->eventDispatcher->dispatch(new MutationTestingWasStarted($numberOfMutants));
 
+        $notMatchedSourceCodeRegexes = array_flip(array_unique(array_merge(...array_values($this->ignoreSourceCodeMutatorsMap))));
+
         $processes = take($mutations)
             ->cast(function (Mutation $mutation): Mutant {
                 return $this->mutantFactory->create($mutation);
             })
-            ->filter(function (Mutant $mutant): bool {
+            ->filter(function (Mutant $mutant) use (&$notMatchedSourceCodeRegexes): bool {
                 $mutatorName = $mutant->getMutation()->getMutatorName();
 
                 foreach ($this->ignoreSourceCodeMutatorsMap[$mutatorName] ?? [] as $sourceCodeRegex) {
                     if ($this->diffSourceCodeMatcher->matches($mutant->getDiff()->get(), $sourceCodeRegex)) {
+                        unset($notMatchedSourceCodeRegexes[$sourceCodeRegex]);
+
                         $this->eventDispatcher->dispatch(new MutantProcessWasFinished(
                             MutantExecutionResult::createFromIgnoredMutant($mutant)
                         ));
@@ -118,6 +127,13 @@ public function run(iterable $mutations, string $testFrameworkExtraOptions): voi
 
                 return true;
             })
+            ->filter(static function (Mutant $mutant) use ($notMatchedSourceCodeRegexes): bool {
+                if ($notMatchedSourceCodeRegexes !== []) {
+                    throw NotMatchedIgnoreSourceCodeRegexFound::forRegexes(array_keys($notMatchedSourceCodeRegexes));
+                }
+
+                return true;
+            })
             ->filter(function (Mutant $mutant): bool {
                 // It's a proxy call to Mutation, can be done one stage up
                 if ($mutant->isCoveredByTest()) {

src/Process/Runner/NotMatchedIgnoreSourceCodeRegexFound.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * This code is licensed under the BSD 3-Clause License.
+ *
+ * Copyright (c) 2017, Maks Rafalko
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+declare(strict_types=1);
+
+namespace Infection\Process\Runner;
+
+use Exception;
+use function implode;
+
+/**
+ * @internal
+ */
+final class NotMatchedIgnoreSourceCodeRegexFound extends Exception
+{
+    /**
+     * @param array<int, string> $notMatchedSourceCodeRegexes
+     */
+    public static function forRegexes(array $notMatchedSourceCodeRegexes): self
+    {
+        return new self('The following ignore source code regexes were not matched, consider removing them from `infection.json`: ' . implode(', ', $notMatchedSourceCodeRegexes));
+    }
+}

tests/e2e/Example_Test/infection.json

@@ -9,5 +9,14 @@
     "logs": {
         "summary": "infection.log"
     },
-    "tmpDir": "."
+    "tmpDir": ".",
+    "mutators": {
+        "@default": false,
+        "ProtectedVisibility": true,
+        "Plus": true,
+        "global-ignoreSourceCodeByRegex": [
+            "public(.*)"
+        ],
+        "PublicVisibility": true
+    }
 }

tests/phpunit/Process/Runner/MutationTestingRunnerTest.php

@@ -53,6 +53,7 @@
 use Infection\Process\Factory\MutantProcessFactory;
 use Infection\Process\MutantProcess;
 use Infection\Process\Runner\MutationTestingRunner;
+use Infection\Process\Runner\NotMatchedIgnoreSourceCodeRegexFound;
 use Infection\Process\Runner\ProcessRunner;
 use Infection\Tests\Fixtures\Event\EventDispatcherCollector;
 use Infection\Tests\Mutant\MutantBuilder;
@@ -371,6 +372,75 @@ public function test_it_does_not_create_processes_when_code_is_ignored_by_regex(
         );
     }
 
+    public function test_fails_with_error_when_not_all_ignore_source_code_regexes_are_used(): void
+    {
+        $mutations = [
+            $mutation0 = $this->createMutation(0),
+        ];
+
+        $testFrameworkExtraOptions = '--filter=acme/FooTest.php';
+
+        $mutant = MutantBuilder::build(
+            '/path/to/mutant0',
+            $mutation0,
+            'mutated code 0',
+            '- Non matching source code line',
+            '<?php $a = 1;'
+        );
+
+        $this->mutantFactoryMock
+            ->method('create')
+            ->withConsecutive(
+                [$mutation0],
+            )
+            ->willReturnOnConsecutiveCalls($mutant)
+        ;
+
+        $this->fileSystemMock
+            ->expects($this->never())
+            ->method($this->anything())
+        ;
+
+        $this->processFactoryMock
+            ->expects($this->never())
+            ->method($this->anything())
+        ;
+
+        $this->processRunnerMock
+            ->expects($this->never())
+            ->method('run')
+            ->with($this->emptyIterable())
+        ;
+
+        $this->runner = new MutationTestingRunner(
+            $this->processFactoryMock,
+            $this->mutantFactoryMock,
+            $this->processRunnerMock,
+            $this->eventDispatcher,
+            $this->fileSystemMock,
+            new DiffSourceCodeMatcher(),
+            true,
+            100.0,
+            [
+                'For_' => ['Assert::.*'],
+            ]
+        );
+
+        $this->expectException(NotMatchedIgnoreSourceCodeRegexFound::class);
+        $this->expectExceptionMessage('The following ignore source code regexes were not matched, consider removing them from `infection.json`: Assert::.*');
+
+        $this->runner->run($mutations, $testFrameworkExtraOptions);
+
+        $this->assertAreSameEvents(
+            [
+                new MutationTestingWasStarted(0),
+                new MutantProcessWasFinished(MutantExecutionResult::createFromNonCoveredMutant($mutant)),
+                new MutationTestingWasFinished(),
+            ],
+            $this->eventDispatcher->getEvents()
+        );
+    }
+
     public function test_it_passes_through_iterables_when_concurrent_execution_requested(): void
     {
         $mutations = new ArrayIterator();

tests/phpunit/Process/Runner/NotMatchedIgnoreSourceCodeRegexFoundTest.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * This code is licensed under the BSD 3-Clause License.
+ *
+ * Copyright (c) 2017, Maks Rafalko
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+declare(strict_types=1);
+
+namespace Infection\Tests\Process\Runner;
+
+use Infection\Process\Runner\NotMatchedIgnoreSourceCodeRegexFound;
+use PHPUnit\Framework\TestCase;
+
+final class NotMatchedIgnoreSourceCodeRegexFoundTest extends TestCase
+{
+    public function test_it_is_being_created_with_a_message(): void
+    {
+        $exception = NotMatchedIgnoreSourceCodeRegexFound::forRegexes(['a', 'b']);
+
+        $this->assertInstanceOf(NotMatchedIgnoreSourceCodeRegexFound::class, $exception);
+        $this->assertSame(
+            'The following ignore source code regexes were not matched, consider removing them from `infection.json`: a, b',
+            $exception->getMessage()
+        );
+    }
+}