feat(reload test environment): implement test environment reload (#3369)

Add the ability for Stryker to restart a test runner in order to reload its test environment. This is used to test static mutants where the test runner is incapable of reloading the test environment by itself. This is a requirement to support es modules (ESM) since it is impossible to reload an ESM using something like `delete require.cache['...']`. Instead, Stryker will restart the entire test runner process for those mutants.

The way for the test runner to communicate to Stryker whether it is capable of reloading the environment is by implementing the new `capabilities()` method.

```ts
capabilities() {
  return { reloadEnvironment: true }
}
```

If you do this, you can expect the new `reloadEnvironment` boolean in your `MutantRunOptions` to be true when you need to reload the test environment after activating the mutant.

BREAKING CHANGE: Test runner plugins must provide `TestRunnerCapabilities` by implementing the `capabilities` method.

packages/api/src/core/mutant.ts

@@ -25,12 +25,7 @@ export interface Mutant extends Pick<schema.MutantResult, 'id' | 'location' | 'm
 /**
  * Represents a mutant in its matched-with-the-tests state, ready to be tested.
  */
-export type MutantTestCoverage = Mutant &
-  Pick<schema.MutantResult, 'coveredBy' | 'static'> & {
-    estimatedNetTime: number;
-    hitCount?: number;
-    testFilter: string[] | undefined;
-  };
+export type MutantTestCoverage = Mutant & Pick<schema.MutantResult, 'coveredBy' | 'static'>;
 
 /**
  * Represents a mutant in its final state, ready to be reported.

packages/api/src/test-runner/index.ts

@@ -6,3 +6,4 @@ export * from './run-options';
 export * from './mutant-run-result';
 export * from './dry-run-status';
 export * from './run-result-helpers';
+export * from './test-runner-capabilities';

packages/api/src/test-runner/run-options.ts

@@ -27,4 +27,11 @@ export interface MutantRunOptions extends RunOptions {
   hitLimit?: number;
   activeMutant: Mutant;
   sandboxFileName: string;
+  /**
+   * Determines whether or not the test environment should be reloaded.
+   * This is necessary when testing static mutants, where the mutant is only executed when the test environment is loaded.
+   * A test runner might be unable to reload the test environment, i.e. when the files were loaded via `import` in nodejs.
+   * In which case the test runner should report `reloadEnvironment: false` in it's capabilities.
+   */
+  reloadEnvironment: boolean;
 }

packages/api/src/test-runner/test-runner-capabilities.ts

@@ -0,0 +1,11 @@
+/**
+ * Represents the capabilities of a test runner.
+ */
+export interface TestRunnerCapabilities {
+  /**
+   * When true, the test runner is capable of reloading the test environment. Otherwise false.
+   * Reloading means creating a new nodejs process, or reloading the browser.
+   * When true, the test runner should reload the test environment when `reloadEnvironment` is present in the run options.
+   */
+  reloadEnvironment: boolean;
+}

packages/api/src/test-runner/test-runner.ts

@@ -1,8 +1,10 @@
 import { DryRunOptions, MutantRunOptions } from './run-options';
 import { DryRunResult } from './dry-run-result';
 import { MutantRunResult } from './mutant-run-result';
+import { TestRunnerCapabilities } from './test-runner-capabilities';
 
 export interface TestRunner {
+  capabilities(): Promise<TestRunnerCapabilities> | TestRunnerCapabilities;
   init?(): Promise<void>;
   dryRun(options: DryRunOptions): Promise<DryRunResult>;
   mutantRun(options: MutantRunOptions): Promise<MutantRunResult>;

packages/core/src/di/core-tokens.ts

@@ -9,7 +9,7 @@ export const inputFiles = 'inputFiles';
 export const dryRunResult = 'dryRunResult';
 export const files = 'files';
 export const mutants = 'mutants';
-export const mutantsWithTestCoverage = 'mutantsWithTestCoverage';
+export const mutantTestPlanner = 'mutantTestPlanner';
 export const process = 'process';
 export const temporaryDirectory = 'temporaryDirectory';
 export const unexpectedExitRegistry = 'unexpectedExitRegistry';

packages/core/src/mutants/index.ts

@@ -1 +1,2 @@
-export * from './find-mutant-test-coverage';
+export * from './mutant-test-plan';
+export * from './mutant-test-planner';

packages/core/src/mutants/mutant-test-plan.ts

@@ -0,0 +1,23 @@
+import { MutantStatus, MutantTestCoverage } from '@stryker-mutator/api/core';
+import { MutantRunOptions } from '@stryker-mutator/api/test-runner';
+
+/**
+ * Represents an internal object
+ */
+export type MutantTestPlan = MutantEarlyResultPlan | MutantRunPlan;
+
+export enum PlanKind {
+  EarlyResult = 'EarlyResult',
+  Run = 'Run',
+}
+
+export interface MutantEarlyResultPlan {
+  plan: PlanKind.EarlyResult;
+  mutant: MutantTestCoverage & { status: MutantStatus };
+}
+
+export interface MutantRunPlan {
+  plan: PlanKind.Run;
+  mutant: MutantTestCoverage;
+  runOptions: MutantRunOptions;
+}