From fbe3ce4f7d5c27fec3fa1e32ee86b23b169e02d2 Mon Sep 17 00:00:00 2001 From: Nico Jansen Date: Mon, 11 May 2020 21:55:24 +0200 Subject: [PATCH] Add ability to run tests in a mocha instance multiple times (#4234); closes #2783 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ability to run tests in a mocha instance multiple times * Rename `autoDispsoe` to `cleanReferencesAfterRun`, Rename since we cannot dispose the entire mocha instance after a test run. We should keep the process.on('uncaughtException') handlers in place in order to not break existing functionality. * Allow `unloadFiles` to reset `referencesCleaned`. * Complete rename of _cleanReferencesAfterRun * Add integration test for running a suite multiple times * improve api docs * Docs: fix dead link * Make sure tests run on older node versions * Remove `.only` 😅 * Implement `process.listenerCount` in the browser * Implement mocha states in a finite state machine * Fix some small remarks * Make integration tests more damp * Keep `Runner` api backward compatible * Unload files when disposed * Runnable.reset should also reset `err` and `state` * Also reset hooks Co-authored-by: Christopher Hiller --- browser-entry.js | 11 ++ lib/errors.js | 33 +++- lib/hook.js | 8 + lib/mocha.js | 110 +++++++++++- lib/runnable.js | 15 +- lib/runner.js | 88 +++++++-- lib/suite.js | 32 +++- lib/test.js | 13 +- .../multiple-runs/clean-references.fixture.js | 6 + .../fixtures/multiple-runs/dispose.fixture.js | 6 + ...uns-with-different-output-suite.fixture.js | 19 ++ ...ns-with-flaky-before-each-suite.fixture.js | 18 ++ ...ple-runs-with-flaky-before-each.fixture.js | 13 ++ .../multiple-runs/run-thrice-helper.js | 24 +++ .../multiple-runs/run-thrice.fixture.js | 6 + ...previous-is-still-running-suite.fixture.js | 5 + ...un-if-previous-is-still-running.fixture.js | 12 ++ test/integration/multiple-runs.spec.js | 89 ++++++++++ test/unit/hook.spec.js | 44 +++++ test/unit/mocha.spec.js | 167 ++++++++++++++++++ test/unit/runnable.spec.js | 23 +++ test/unit/runner.spec.js | 61 ++++++- test/unit/suite.spec.js | 42 +++++ test/unit/test.spec.js | 34 +++- 24 files changed, 853 insertions(+), 26 deletions(-) create mode 100644 test/integration/fixtures/multiple-runs/clean-references.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/dispose.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/multiple-runs-with-different-output-suite.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each-suite.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/run-thrice-helper.js create mode 100644 test/integration/fixtures/multiple-runs/run-thrice.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running-suite.fixture.js create mode 100644 test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running.fixture.js create mode 100644 test/integration/multiple-runs.spec.js create mode 100644 test/unit/hook.spec.js diff --git a/browser-entry.js b/browser-entry.js index 3e9cbbaf90..114d2f7213 100644 --- a/browser-entry.js +++ b/browser-entry.js @@ -52,6 +52,17 @@ process.removeListener = function(e, fn) { } }; +/** + * Implements listenerCount for 'uncaughtException'. + */ + +process.listenerCount = function(name) { + if (name === 'uncaughtException') { + return uncaughtExceptionHandlers.length; + } + return 0; +}; + /** * Implements uncaughtException listener. */ diff --git a/lib/errors.js b/lib/errors.js index 099bc579ab..a85c8c24b9 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -149,6 +149,35 @@ function createInvalidPluginError(message, pluginType, pluginId) { } } +/** + * Creates an error object to be thrown when a mocha object's `run` method is executed while it is already disposed. + * @param {string} message The error message to be displayed. + * @param {boolean} cleanReferencesAfterRun the value of `cleanReferencesAfterRun` + * @param {Mocha} instance the mocha instance that throw this error + */ +function createMochaInstanceAlreadyDisposedError( + message, + cleanReferencesAfterRun, + instance +) { + var err = new Error(message); + err.code = 'ERR_MOCHA_INSTANCE_ALREADY_DISPOSED'; + err.cleanReferencesAfterRun = cleanReferencesAfterRun; + err.instance = instance; + return err; +} + +/** + * Creates an error object to be thrown when a mocha object's `run` method is called while a test run is in progress. + * @param {string} message The error message to be displayed. + */ +function createMochaInstanceAlreadyRunningError(message, instance) { + var err = new Error(message); + err.code = 'ERR_MOCHA_INSTANCE_ALREADY_RUNNING'; + err.instance = instance; + return err; +} + module.exports = { createInvalidArgumentTypeError: createInvalidArgumentTypeError, createInvalidArgumentValueError: createInvalidArgumentValueError, @@ -158,5 +187,7 @@ module.exports = { createMissingArgumentError: createMissingArgumentError, createNoFilesMatchPatternError: createNoFilesMatchPatternError, createUnsupportedError: createUnsupportedError, - createInvalidPluginError: createInvalidPluginError + createInvalidPluginError: createInvalidPluginError, + createMochaInstanceAlreadyDisposedError: createMochaInstanceAlreadyDisposedError, + createMochaInstanceAlreadyRunningError: createMochaInstanceAlreadyRunningError }; diff --git a/lib/hook.js b/lib/hook.js index 71440d23d0..6560715fc5 100644 --- a/lib/hook.js +++ b/lib/hook.js @@ -27,6 +27,14 @@ function Hook(title, fn) { */ inherits(Hook, Runnable); +/** + * Resets the state for a next run. + */ +Hook.prototype.reset = function() { + Runnable.prototype.reset.call(this); + delete this._error; +}; + /** * Get or set the test `err`. * diff --git a/lib/mocha.js b/lib/mocha.js index d7d1d54709..5a8fb32202 100644 --- a/lib/mocha.js +++ b/lib/mocha.js @@ -18,6 +18,10 @@ var esmUtils = utils.supportsEsModules() ? require('./esm-utils') : undefined; var createStatsCollector = require('./stats-collector'); var createInvalidReporterError = errors.createInvalidReporterError; var createInvalidInterfaceError = errors.createInvalidInterfaceError; +var createMochaInstanceAlreadyDisposedError = + errors.createMochaInstanceAlreadyDisposedError; +var createMochaInstanceAlreadyRunningError = + errors.createMochaInstanceAlreadyRunningError; var EVENT_FILE_PRE_REQUIRE = Suite.constants.EVENT_FILE_PRE_REQUIRE; var EVENT_FILE_POST_REQUIRE = Suite.constants.EVENT_FILE_POST_REQUIRE; var EVENT_FILE_REQUIRE = Suite.constants.EVENT_FILE_REQUIRE; @@ -25,6 +29,30 @@ var sQuote = utils.sQuote; exports = module.exports = Mocha; +/** + * A Mocha instance is a finite state machine. + * These are the states it can be in. + */ +var mochaStates = utils.defineConstants({ + /** + * Initial state of the mocha instance + */ + INIT: 'init', + /** + * Mocha instance is running tests + */ + RUNNING: 'running', + /** + * Mocha instance is done running tests and references to test functions and hooks are cleaned. + * You can reset this state by unloading the test files. + */ + REFERENCES_CLEANED: 'referencesCleaned', + /** + * Mocha instance is disposed and can no longer be used. + */ + DISPOSED: 'disposed' +}); + /** * To require local UIs and reporters when running in node. */ @@ -97,6 +125,7 @@ function Mocha(options) { this.options = options; // root suite this.suite = new exports.Suite('', new exports.Context(), true); + this._cleanReferencesAfterRun = true; this.grep(options.grep) .fgrep(options.fgrep) @@ -388,9 +417,18 @@ Mocha.unloadFile = function(file) { * @chainable */ Mocha.prototype.unloadFiles = function() { + if (this._state === mochaStates.DISPOSED) { + throw createMochaInstanceAlreadyDisposedError( + 'Mocha instance is already disposed, it cannot be used again.', + this._cleanReferencesAfterRun, + this + ); + } + this.files.forEach(function(file) { Mocha.unloadFile(file); }); + this._state = mochaStates.INIT; return this; }; @@ -490,6 +528,38 @@ Mocha.prototype.checkLeaks = function(checkLeaks) { return this; }; +/** + * Enables or disables whether or not to dispose after each test run. + * Disable this to ensure you can run the test suite multiple times. + * If disabled, be sure to dispose mocha when you're done to prevent memory leaks. + * @public + * @see {@link Mocha#dispose} + * @param {boolean} cleanReferencesAfterRun + * @return {Mocha} this + * @chainable + */ +Mocha.prototype.cleanReferencesAfterRun = function(cleanReferencesAfterRun) { + this._cleanReferencesAfterRun = cleanReferencesAfterRun !== false; + return this; +}; + +/** + * Manually dispose this mocha instance. Mark this instance as `disposed` and unable to run more tests. + * It also removes function references to tests functions and hooks, so variables trapped in closures can be cleaned by the garbage collector. + * @public + */ +Mocha.prototype.dispose = function() { + if (this._state === mochaStates.RUNNING) { + throw createMochaInstanceAlreadyRunningError( + 'Cannot dispose while the mocha instance is still running tests.' + ); + } + this.unloadFiles(); + this._previousRunner && this._previousRunner.dispose(); + this.suite.dispose(); + this._state = mochaStates.DISPOSED; +}; + /** * Displays full stack trace upon test failure. * @@ -770,6 +840,28 @@ Mocha.prototype.forbidPending = function(forbidPending) { return this; }; +/** + * Throws an error if mocha is in the wrong state to be able to transition to a "running" state. + */ +Mocha.prototype._guardRunningStateTransition = function() { + if (this._state === mochaStates.RUNNING) { + throw createMochaInstanceAlreadyRunningError( + 'Mocha instance is currently running tests, cannot start a next test run until this one is done', + this + ); + } + if ( + this._state === mochaStates.DISPOSED || + this._state === mochaStates.REFERENCES_CLEANED + ) { + throw createMochaInstanceAlreadyDisposedError( + 'Mocha instance is already disposed, cannot start a new test run. Please create a new mocha instance. Be sure to set disable `cleanReferencesAfterRun` when you want to reuse the same mocha instance for multiple test runs.', + this._cleanReferencesAfterRun, + this + ); + } +}; + /** * Mocha version as specified by "package.json". * @@ -810,13 +902,23 @@ Object.defineProperty(Mocha.prototype, 'version', { * mocha.run(failures => process.exitCode = failures ? 1 : 0); */ Mocha.prototype.run = function(fn) { + this._guardRunningStateTransition(); + this._state = mochaStates.RUNNING; + if (this._previousRunner) { + this._previousRunner.dispose(); + this.suite.reset(); + } if (this.files.length && !this.loadAsync) { this.loadFiles(); } + var self = this; var suite = this.suite; var options = this.options; options.files = this.files; - var runner = new exports.Runner(suite, options.delay); + var runner = new exports.Runner(suite, { + delay: options.delay, + cleanReferencesAfterRun: this._cleanReferencesAfterRun + }); createStatsCollector(runner); var reporter = new this._reporter(runner, options); runner.checkLeaks = options.checkLeaks === true; @@ -841,6 +943,12 @@ Mocha.prototype.run = function(fn) { exports.reporters.Base.hideDiff = !options.diff; function done(failures) { + self._previousRunner = runner; + if (self._cleanReferencesAfterRun) { + self._state = mochaStates.REFERENCES_CLEANED; + } else { + self._state = mochaStates.INIT; + } fn = fn || utils.noop; if (reporter.done) { reporter.done(failures, fn); diff --git a/lib/runnable.js b/lib/runnable.js index ed585eb93f..4d58070f5d 100644 --- a/lib/runnable.js +++ b/lib/runnable.js @@ -35,10 +35,8 @@ function Runnable(title, fn) { this.sync = !this.async; this._timeout = 2000; this._slow = 75; - this.timedOut = false; this._retries = -1; - this._currentRetry = 0; - this.pending = false; + this.reset(); } /** @@ -46,6 +44,17 @@ function Runnable(title, fn) { */ utils.inherits(Runnable, EventEmitter); +/** + * Resets the state initially or for a next run. + */ +Runnable.prototype.reset = function() { + this.timedOut = false; + this._currentRetry = 0; + this.pending = false; + delete this.state; + delete this.err; +}; + /** * Get current timeout value in msecs. * diff --git a/lib/runner.js b/lib/runner.js index aabffda96a..d87c41820d 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -121,18 +121,30 @@ module.exports = Runner; * @extends external:EventEmitter * @public * @class - * @param {Suite} suite Root suite - * @param {boolean} [delay] Whether to delay execution of root suite until ready. + * @param {Suite} suite - Root suite + * @param {Object|boolean} [opts] - Options. If `boolean`, whether or not to delay execution of root suite until ready (for backwards compatibility). + * @param {boolean} [opts.delay] - Whether to delay execution of root suite until ready. + * @param {boolean} [opts.cleanReferencesAfterRun] - Whether to clean references to test fns and hooks when a suite is done. */ -function Runner(suite, delay) { +function Runner(suite, opts) { + if (opts === undefined) { + opts = {}; + } + if (typeof opts === 'boolean') { + this._delay = opts; + opts = {}; + } else { + this._delay = opts.delay; + } var self = this; this._globals = []; this._abort = false; - this._delay = delay; this.suite = suite; this.started = false; + this._opts = opts; this.total = suite.total(); this.failures = 0; + this._eventListeners = []; this.on(constants.EVENT_TEST_END, function(test) { if (test.type === 'test' && test.retriedTest() && test.parent) { var idx = @@ -162,6 +174,53 @@ Runner.immediately = global.setImmediate || process.nextTick; */ inherits(Runner, EventEmitter); +/** + * Replacement for `target.on(eventName, listener)` that does bookkeeping to remove them when this runner instance is disposed. + * @param target {EventEmitter} + * @param eventName {string} + * @param fn {function} + */ +Runner.prototype._addEventListener = function(target, eventName, listener) { + target.on(eventName, listener); + this._eventListeners.push([target, eventName, listener]); +}; + +/** + * Replacement for `target.removeListener(eventName, listener)` that also updates the bookkeeping. + * @param target {EventEmitter} + * @param eventName {string} + * @param fn {function} + */ +Runner.prototype._removeEventListener = function(target, eventName, listener) { + var eventListenerIndex = this._eventListeners.findIndex(function( + eventListenerDescriptor + ) { + return ( + eventListenerDescriptor[0] === target && + eventListenerDescriptor[1] === eventName && + eventListenerDescriptor[2] === listener + ); + }); + if (eventListenerIndex !== -1) { + var removedListener = this._eventListeners.splice(eventListenerIndex, 1)[0]; + removedListener[0].removeListener(removedListener[1], removedListener[2]); + } +}; + +/** + * Removes all event handlers set during a run on this instance. + * Remark: this does _not_ clean/dispose the tests or suites themselves. + */ +Runner.prototype.dispose = function() { + this.removeAllListeners(); + this._eventListeners.forEach(function(eventListenerDescriptor) { + eventListenerDescriptor[0].removeListener( + eventListenerDescriptor[1], + eventListenerDescriptor[2] + ); + }); +}; + /** * Run tests with full titles matching `re`. Updates runner.total * with number of tests matched. @@ -378,7 +437,7 @@ Runner.prototype.hook = function(name, fn) { self.emit(constants.EVENT_HOOK_BEGIN, hook); if (!hook.listeners('error').length) { - hook.on('error', function(err) { + self._addEventListener(hook, 'error', function(err) { self.failHook(hook, err); }); } @@ -530,7 +589,7 @@ Runner.prototype.runTest = function(fn) { if (this.asyncOnly) { test.asyncOnly = true; } - test.on('error', function(err) { + this._addEventListener(test, 'error', function(err) { self.fail(test, err); }); if (this.allowUncaught) { @@ -920,21 +979,24 @@ Runner.prototype.run = function(fn) { } // references cleanup to avoid memory leaks - this.on(constants.EVENT_SUITE_END, function(suite) { - suite.cleanReferences(); - }); + if (this._opts.cleanReferencesAfterRun) { + this.on(constants.EVENT_SUITE_END, function(suite) { + suite.cleanReferences(); + }); + } // callback this.on(constants.EVENT_RUN_END, function() { - process.removeListener('uncaughtException', uncaught); - process.on('uncaughtException', self.uncaughtEnd); + debug(constants.EVENT_RUN_END); + self._removeEventListener(process, 'uncaughtException', uncaught); + self._addEventListener(process, 'uncaughtException', self.uncaughtEnd); debug('run(): emitted %s', constants.EVENT_RUN_END); fn(self.failures); }); // uncaught exception - process.removeListener('uncaughtException', self.uncaughtEnd); - process.on('uncaughtException', uncaught); + self._removeEventListener(process, 'uncaughtException', self.uncaughtEnd); + self._addEventListener(process, 'uncaughtException', uncaught); if (this._delay) { // for reporters, I guess. diff --git a/lib/suite.js b/lib/suite.js index 2a64e2db8f..dc42fd74fd 100644 --- a/lib/suite.js +++ b/lib/suite.js @@ -61,19 +61,19 @@ function Suite(title, parentContext, isRoot) { this.ctx = new Context(); this.suites = []; this.tests = []; + this.root = isRoot === true; this.pending = false; + this._retries = -1; this._beforeEach = []; this._beforeAll = []; this._afterEach = []; this._afterAll = []; - this.root = isRoot === true; this._timeout = 2000; this._slow = 75; this._bail = false; - this._retries = -1; this._onlyTests = []; this._onlySuites = []; - this.delayed = false; + this.reset(); this.on('newListener', function(event) { if (deprecatedEvents[event]) { @@ -91,6 +91,22 @@ function Suite(title, parentContext, isRoot) { */ inherits(Suite, EventEmitter); +/** + * Resets the state initially or for a next run. + */ +Suite.prototype.reset = function() { + this.delayed = false; + function doReset(thingToReset) { + thingToReset.reset(); + } + this.suites.forEach(doReset); + this.tests.forEach(doReset); + this._beforeEach.forEach(doReset); + this._afterEach.forEach(doReset); + this._beforeAll.forEach(doReset); + this._afterAll.forEach(doReset); +}; + /** * Return a clone of this `Suite`. * @@ -493,6 +509,16 @@ Suite.prototype.getHooks = function getHooks(name) { return this['_' + name]; }; +/** + * cleans all references from this suite and all child suites. + */ +Suite.prototype.dispose = function() { + this.suites.forEach(function(suite) { + suite.dispose(); + }); + this.cleanReferences(); +}; + /** * Cleans up the references to all the deferred functions * (before/after/beforeEach/afterEach) and tests of a Suite. diff --git a/lib/test.js b/lib/test.js index 29c74b4563..187fe49767 100644 --- a/lib/test.js +++ b/lib/test.js @@ -26,9 +26,9 @@ function Test(title, fn) { 'string' ); } - Runnable.call(this, title, fn); - this.pending = !fn; this.type = 'test'; + Runnable.call(this, title, fn); + this.reset(); } /** @@ -36,6 +36,15 @@ function Test(title, fn) { */ utils.inherits(Test, Runnable); +/** + * Resets the state initially or for a next run. + */ +Test.prototype.reset = function() { + Runnable.prototype.reset.call(this); + this.pending = !this.fn; + delete this.state; +}; + /** * Set or get retried test * diff --git a/test/integration/fixtures/multiple-runs/clean-references.fixture.js b/test/integration/fixtures/multiple-runs/clean-references.fixture.js new file mode 100644 index 0000000000..2f204a0b74 --- /dev/null +++ b/test/integration/fixtures/multiple-runs/clean-references.fixture.js @@ -0,0 +1,6 @@ +'use strict'; +const Mocha = require('../../../../lib/mocha'); + +const mocha = new Mocha({ reporter: 'json' }); +mocha.cleanReferencesAfterRun(true); +require('./run-thrice-helper')(mocha); diff --git a/test/integration/fixtures/multiple-runs/dispose.fixture.js b/test/integration/fixtures/multiple-runs/dispose.fixture.js new file mode 100644 index 0000000000..c0d3c4d7ba --- /dev/null +++ b/test/integration/fixtures/multiple-runs/dispose.fixture.js @@ -0,0 +1,6 @@ +'use strict'; +const Mocha = require('../../../../lib/mocha'); + +const mocha = new Mocha({ reporter: 'json' }); +mocha.dispose(); +require('./run-thrice-helper')(mocha); diff --git a/test/integration/fixtures/multiple-runs/multiple-runs-with-different-output-suite.fixture.js b/test/integration/fixtures/multiple-runs/multiple-runs-with-different-output-suite.fixture.js new file mode 100644 index 0000000000..903f661bf9 --- /dev/null +++ b/test/integration/fixtures/multiple-runs/multiple-runs-with-different-output-suite.fixture.js @@ -0,0 +1,19 @@ +describe('Multiple runs', () => { + + /** + * Shared state! Bad practice, but nice for this test + */ + let i = 0; + + it('should skip, fail and pass respectively', function () { + switch (i++) { + case 0: + this.skip(); + case 1: + throw new Error('Expected error'); + default: + // this is fine ☕ + break; + } + }); +}); diff --git a/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each-suite.fixture.js b/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each-suite.fixture.js new file mode 100644 index 0000000000..7863fb223e --- /dev/null +++ b/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each-suite.fixture.js @@ -0,0 +1,18 @@ +describe('Multiple runs', () => { + + /** + * Shared state! Bad practice, but nice for this test + */ + let i = 0; + + beforeEach(function () { + if (i++ === 0) { + throw new Error('Expected error for this test'); + } + }); + + + it('should be a dummy test', function () { + // this is fine ☕ + }); +}); diff --git a/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each.fixture.js b/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each.fixture.js new file mode 100644 index 0000000000..1a4707705f --- /dev/null +++ b/test/integration/fixtures/multiple-runs/multiple-runs-with-flaky-before-each.fixture.js @@ -0,0 +1,13 @@ +'use strict'; +const Mocha = require('../../../../lib/mocha'); + +const mocha = new Mocha({ reporter: 'json' }); +mocha.cleanReferencesAfterRun(false); +mocha.addFile(require.resolve('./multiple-runs-with-flaky-before-each-suite.fixture.js')); +console.log('['); +mocha.run(() => { + console.log(','); + mocha.run(() => { + console.log(']'); + }); +}); diff --git a/test/integration/fixtures/multiple-runs/run-thrice-helper.js b/test/integration/fixtures/multiple-runs/run-thrice-helper.js new file mode 100644 index 0000000000..58f2c9de5e --- /dev/null +++ b/test/integration/fixtures/multiple-runs/run-thrice-helper.js @@ -0,0 +1,24 @@ +module.exports = function (mocha) { + mocha.addFile(require.resolve('./multiple-runs-with-different-output-suite.fixture.js')); + console.log('['); + try { + mocha.run(() => { + console.log(','); + try { + mocha.run(() => { + console.log(','); + mocha.run(() => { + console.log(']'); + }); + }); + } catch (err) { + console.error(err.code); + throw err; + } + }); + } catch (err) { + console.error(err.code); + throw err; + } + +} diff --git a/test/integration/fixtures/multiple-runs/run-thrice.fixture.js b/test/integration/fixtures/multiple-runs/run-thrice.fixture.js new file mode 100644 index 0000000000..3c63ec3725 --- /dev/null +++ b/test/integration/fixtures/multiple-runs/run-thrice.fixture.js @@ -0,0 +1,6 @@ +'use strict'; +const Mocha = require('../../../../lib/mocha'); + +const mocha = new Mocha({ reporter: 'json' }); +mocha.cleanReferencesAfterRun(false); +require('./run-thrice-helper')(mocha); diff --git a/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running-suite.fixture.js b/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running-suite.fixture.js new file mode 100644 index 0000000000..a8ecaf76e5 --- /dev/null +++ b/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running-suite.fixture.js @@ -0,0 +1,5 @@ +describe('slow suite', () => { + it('should be slow', (done) => { + setTimeout(200, done); + }); +}); diff --git a/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running.fixture.js b/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running.fixture.js new file mode 100644 index 0000000000..1b031334b1 --- /dev/null +++ b/test/integration/fixtures/multiple-runs/start-second-run-if-previous-is-still-running.fixture.js @@ -0,0 +1,12 @@ +'use strict'; +const Mocha = require('../../../../lib/mocha'); + +const mocha = new Mocha({ reporter: 'json' }); +mocha.addFile(require.resolve('./start-second-run-if-previous-is-still-running-suite.fixture.js')); +mocha.run(); +try { + mocha.run(); +} catch (err) { + console.error(err.code); +} + diff --git a/test/integration/multiple-runs.spec.js b/test/integration/multiple-runs.spec.js new file mode 100644 index 0000000000..61d672d4b2 --- /dev/null +++ b/test/integration/multiple-runs.spec.js @@ -0,0 +1,89 @@ +'use strict'; + +var invokeNode = require('./helpers').invokeNode; + +describe('multiple runs', function(done) { + it('should be allowed to run multiple times if cleanReferences is turned off', function(done) { + var path = require.resolve( + './fixtures/multiple-runs/run-thrice.fixture.js' + ); + invokeNode([path], function(err, res) { + expect(err, 'to be null'); + expect(res.code, 'to be', 0); + var results = JSON.parse(res.output); + expect(results, 'to have length', 3); + expect(results[0].pending, 'to have length', 1); + expect(results[0].failures, 'to have length', 0); + expect(results[0].passes, 'to have length', 0); + expect(results[1].pending, 'to have length', 0); + expect(results[1].failures, 'to have length', 1); + expect(results[1].passes, 'to have length', 0); + expect(results[2].pending, 'to have length', 0); + expect(results[2].failures, 'to have length', 0); + expect(results[2].passes, 'to have length', 1); + done(); + }); + }); + + it('should not be allowed if cleanReferences is true', function(done) { + var path = require.resolve( + './fixtures/multiple-runs/clean-references.fixture.js' + ); + invokeNode( + [path], + function(err, res) { + expect(err, 'to be null'); + expect(res.code, 'not to be', 0); + expect(res.output, 'to contain', 'ERR_MOCHA_INSTANCE_ALREADY_DISPOSED'); + done(); + }, + {stdio: ['ignore', 'pipe', 'pipe']} + ); + }); + + it('should not be allowed if the instance is disposed', function(done) { + var path = require.resolve('./fixtures/multiple-runs/dispose.fixture.js'); + invokeNode( + [path, '--directly-dispose'], + function(err, res) { + expect(err, 'to be null'); + expect(res.code, 'not to be', 0); + expect(res.output, 'to contain', 'ERR_MOCHA_INSTANCE_ALREADY_DISPOSED'); + done(); + }, + {stdio: ['ignore', 'pipe', 'pipe']} + ); + }); + + it('should not be allowed to run while a previous run is in progress', function(done) { + var path = require.resolve( + './fixtures/multiple-runs/start-second-run-if-previous-is-still-running.fixture' + ); + invokeNode( + [path], + function(err, res) { + expect(err, 'to be null'); + expect(res.output, 'to contain', 'ERR_MOCHA_INSTANCE_ALREADY_RUNNING'); + done(); + }, + {stdio: ['ignore', 'pipe', 'pipe']} + ); + }); + + it('should reset the hooks between runs', function(done) { + var path = require.resolve( + './fixtures/multiple-runs/multiple-runs-with-flaky-before-each.fixture' + ); + invokeNode([path], function(err, res) { + expect(err, 'to be null'); + expect(res.code, 'to be', 0); + var results = JSON.parse(res.output); + expect(results, 'to have length', 2); + expect(results[0].failures, 'to have length', 1); + expect(results[0].passes, 'to have length', 0); + expect(results[1].passes, 'to have length', 1); + expect(results[1].failures, 'to have length', 0); + done(); + }); + }); +}); diff --git a/test/unit/hook.spec.js b/test/unit/hook.spec.js new file mode 100644 index 0000000000..b02a6c5120 --- /dev/null +++ b/test/unit/hook.spec.js @@ -0,0 +1,44 @@ +'use strict'; +var sinon = require('sinon'); +var Mocha = require('../../lib/mocha'); +var Hook = Mocha.Hook; +var Runnable = Mocha.Runnable; + +describe(Hook.name, function() { + var hook; + + beforeEach(function() { + hook = new Hook('Some hook', function() {}); + }); + + afterEach(function() { + sinon.restore(); + }); + + describe('error', function() { + it('should set the hook._error', function() { + var expectedError = new Error('Expected error'); + hook.error(expectedError); + expect(hook._error, 'to be', expectedError); + }); + it('should get the hook._error when called without arguments', function() { + var expectedError = new Error('Expected error'); + hook._error = expectedError; + expect(hook.error(), 'to be', expectedError); + }); + }); + + describe('reset', function() { + it('should call Runnable.reset', function() { + var runnableResetStub = sinon.stub(Runnable.prototype, 'reset'); + hook.reset(); + expect(runnableResetStub, 'was called once'); + }); + + it('should reset the error state', function() { + hook.error(new Error('Expected error for test')); + hook.reset(); + expect(hook.error(), 'to be undefined'); + }); + }); +}); diff --git a/test/unit/mocha.spec.js b/test/unit/mocha.spec.js index 8839a2d0ed..c676d5f96b 100644 --- a/test/unit/mocha.spec.js +++ b/test/unit/mocha.spec.js @@ -22,6 +22,10 @@ describe('Mocha', function() { sandbox.stub(Mocha.prototype, 'global').returnsThis(); }); + it('should set _cleanReferencesAfterRun to true', function() { + expect(new Mocha()._cleanReferencesAfterRun, 'to be', true); + }); + describe('when "options.timeout" is `undefined`', function() { it('should not attempt to set timeout', function() { // eslint-disable-next-line no-new @@ -127,6 +131,25 @@ describe('Mocha', function() { }); }); + describe('#cleanReferencesAfterRun()', function() { + it('should set the _cleanReferencesAfterRun attribute', function() { + var mocha = new Mocha(opts); + mocha.cleanReferencesAfterRun(); + expect(mocha._cleanReferencesAfterRun, 'to be', true); + }); + + it('should set the _cleanReferencesAfterRun attribute to false', function() { + var mocha = new Mocha(opts); + mocha.cleanReferencesAfterRun(false); + expect(mocha._cleanReferencesAfterRun, 'to be', false); + }); + + it('should be chainable', function() { + var mocha = new Mocha(opts); + expect(mocha.cleanReferencesAfterRun(), 'to be', mocha); + }); + }); + describe('#color()', function() { it('should set the color option to true', function() { var mocha = new Mocha(opts); @@ -178,6 +201,32 @@ describe('Mocha', function() { }); }); + describe('#dispose()', function() { + it('should dispose the root suite', function() { + var mocha = new Mocha(opts); + var disposeStub = sandbox.stub(mocha.suite, 'dispose'); + mocha.dispose(); + expect(disposeStub, 'was called once'); + }); + + it('should dispose previous test runner', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + var disposeStub = sandbox.stub(Mocha.Runner.prototype, 'dispose'); + mocha.run(); + runStub.callArg(0); + mocha.dispose(); + expect(disposeStub, 'was called once'); + }); + + it('should unload the files', function() { + var mocha = new Mocha(opts); + var unloadFilesStub = sandbox.stub(mocha, 'unloadFiles'); + mocha.dispose(); + expect(unloadFilesStub, 'was called once'); + }); + }); + describe('#forbidOnly()', function() { it('should set the forbidOnly option to true', function() { var mocha = new Mocha(opts); @@ -434,6 +483,99 @@ describe('Mocha', function() { mocha.run().on('end', done); }); + it('should throw if a run is in progress', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + mocha.run(); + expect( + function() { + mocha.run(); + }, + 'to throw', + { + message: + 'Mocha instance is currently running tests, cannot start a next test run until this one is done', + code: 'ERR_MOCHA_INSTANCE_ALREADY_RUNNING', + instance: mocha + } + ); + expect(runStub, 'was called once'); + }); + + it('should throw the instance is already disposed', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + mocha.dispose(); + expect( + function() { + mocha.run(); + }, + 'to throw', + { + message: + 'Mocha instance is already disposed, cannot start a new test run. Please create a new mocha instance. Be sure to set disable `cleanReferencesAfterRun` when you want to reuse the same mocha instance for multiple test runs.', + code: 'ERR_MOCHA_INSTANCE_ALREADY_DISPOSED', + cleanReferencesAfterRun: true, + instance: mocha + } + ); + expect(runStub, 'was called times', 0); + }); + + it('should throw if a run for a second time', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + mocha.run(); + runStub.callArg(0); + expect( + function() { + mocha.run(); + }, + 'to throw', + { + message: + 'Mocha instance is already disposed, cannot start a new test run. Please create a new mocha instance. Be sure to set disable `cleanReferencesAfterRun` when you want to reuse the same mocha instance for multiple test runs.', + code: 'ERR_MOCHA_INSTANCE_ALREADY_DISPOSED', + instance: mocha + } + ); + expect(runStub, 'was called once'); + }); + + it('should allow multiple runs if `cleanReferencesAfterRun` is disabled', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + mocha.cleanReferencesAfterRun(false); + mocha.run(); + runStub.callArg(0); + mocha.run(); + runStub.callArg(0); + expect(runStub, 'called times', 2); + }); + + it('should reset between runs', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + var resetStub = sandbox.stub(Mocha.Suite.prototype, 'reset'); + mocha.cleanReferencesAfterRun(false); + mocha.run(); + runStub.callArg(0); + mocha.run(); + expect(resetStub, 'was called once'); + }); + + it('should dispose the previous runner when the next run starts', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + var disposeStub = sandbox.stub(Mocha.Runner.prototype, 'dispose'); + mocha.cleanReferencesAfterRun(false); + mocha.run(); + runStub.callArg(0); + expect(disposeStub, 'was not called'); + mocha.run(); + expect(disposeStub, 'was called once'); + }); + describe('#reporter("xunit")#run(fn)', function() { // :TBD: Why does specifying reporter differentiate this test from preceding one it('should not raise errors if callback was not provided', function() { @@ -449,4 +591,29 @@ describe('Mocha', function() { }); }); }); + + describe('#unloadFiles()', function() { + it('should reset referencesCleaned and allow for next run', function() { + var mocha = new Mocha(opts); + var runStub = sandbox.stub(Mocha.Runner.prototype, 'run'); + mocha.run(); + runStub.callArg(0); + mocha.unloadFiles(); + expect(function() { + mocha.run(); + }, 'not to throw'); + }); + + it('should not be allowed when the current instance is already disposed', function() { + var mocha = new Mocha(opts); + mocha.dispose(); + expect( + function() { + mocha.unloadFiles(); + }, + 'to throw', + 'Mocha instance is already disposed, it cannot be used again.' + ); + }); + }); }); diff --git a/test/unit/runnable.spec.js b/test/unit/runnable.spec.js index fa328441ca..bdd2dc145e 100644 --- a/test/unit/runnable.spec.js +++ b/test/unit/runnable.spec.js @@ -127,6 +127,29 @@ describe('Runnable(title, fn)', function() { }); }); + describe('#reset', function() { + var run; + + beforeEach(function() { + run = new Runnable(); + }); + + it('should reset current run state', function() { + run.timedOut = true; + run._currentRetry = 5; + run.pending = true; + run.err = new Error(); + run.state = 'error'; + + run.reset(); + expect(run.timedOut, 'to be false'); + expect(run._currentRetry, 'to be', 0); + expect(run.pending, 'to be false'); + expect(run.err, 'to be undefined'); + expect(run.state, 'to be undefined'); + }); + }); + describe('.title', function() { it('should be present', function() { expect(new Runnable('foo').title, 'to be', 'foo'); diff --git a/test/unit/runner.spec.js b/test/unit/runner.spec.js index c48e2d0e8e..d36d0f2f1f 100644 --- a/test/unit/runner.spec.js +++ b/test/unit/runner.spec.js @@ -15,6 +15,7 @@ var EVENT_TEST_FAIL = Runner.constants.EVENT_TEST_FAIL; var EVENT_TEST_RETRY = Runner.constants.EVENT_TEST_RETRY; var EVENT_TEST_END = Runner.constants.EVENT_TEST_END; var EVENT_RUN_END = Runner.constants.EVENT_RUN_END; +var EVENT_SUITE_END = Runner.constants.EVENT_SUITE_END; var STATE_FAILED = Runnable.constants.STATE_FAILED; describe('Runner', function() { @@ -24,7 +25,7 @@ describe('Runner', function() { beforeEach(function() { suite = new Suite('Suite', 'root'); - runner = new Runner(suite); + runner = new Runner(suite, {cleanReferencesAfterRun: true}); runner.checkLeaks = true; sandbox = sinon.createSandbox(); }); @@ -456,13 +457,69 @@ describe('Runner', function() { done(); }); }); - // karma-mocha is inexplicably doing this with a Hook it('should not throw an exception if something emits EVENT_TEST_END with a non-Test object', function() { expect(function() { runner.emit(EVENT_TEST_END, {}); }, 'not to throw'); }); + + it('should clean references after a run', function() { + runner = new Runner(suite, {delay: false, cleanReferencesAfterRun: true}); + var cleanReferencesStub = sandbox.stub(suite, 'cleanReferences'); + runner.run(); + runner.emit(EVENT_SUITE_END, suite); + expect(cleanReferencesStub, 'was called once'); + }); + + it('should not clean references after a run when `cleanReferencesAfterRun` is `false`', function() { + runner = new Runner(suite, { + delay: false, + cleanReferencesAfterRun: false + }); + var cleanReferencesStub = sandbox.stub(suite, 'cleanReferences'); + runner.run(); + runner.emit(EVENT_SUITE_END, suite); + expect(cleanReferencesStub, 'was not called'); + }); + }); + + describe('.dispose', function() { + it('should remove all listeners from itself', function() { + runner.on('disposeShouldRemoveThis', noop); + runner.dispose(); + expect(runner.listenerCount('disposeShouldRemoveThis'), 'to be', 0); + }); + + it('should remove "error" listeners from a test', function() { + var fn = sandbox.stub(); + runner.test = new Test('test for dispose', fn); + runner.runTest(noop); + // sanity check + expect(runner.test.listenerCount('error'), 'to be', 1); + runner.dispose(); + expect(runner.test.listenerCount('error'), 'to be', 0); + }); + + it('should remove "uncaughtException" listeners from the process', function() { + var normalUncaughtExceptionListenerCount = process.listenerCount( + 'uncaughtException' + ); + sandbox.stub(); + runner.run(noop); + // sanity check + expect( + process.listenerCount('uncaughtException'), + 'to be', + normalUncaughtExceptionListenerCount + 1 + ); + runner.dispose(); + expect( + process.listenerCount('uncaughtException'), + 'to be', + normalUncaughtExceptionListenerCount + ); + }); }); describe('.runTest(fn)', function() { diff --git a/test/unit/suite.spec.js b/test/unit/suite.spec.js index 1be948e1c6..a5063b7f91 100644 --- a/test/unit/suite.spec.js +++ b/test/unit/suite.spec.js @@ -80,6 +80,48 @@ describe('Suite', function() { }); }); + describe('.reset()', function() { + beforeEach(function() { + this.suite = new Suite('Suite to be reset', function() {}); + }); + + it('should reset the `delayed` state', function() { + this.suite.delayed = true; + this.suite.reset(); + expect(this.suite.delayed, 'to be', false); + }); + + it('should forward reset to suites and tests', function() { + var childSuite = new Suite('child suite', this.suite.context); + var test = new Test('test', function() {}); + this.suite.addSuite(childSuite); + this.suite.addTest(test); + var testResetStub = sandbox.stub(test, 'reset'); + var suiteResetStub = sandbox.stub(childSuite, 'reset'); + this.suite.reset(); + expect(testResetStub, 'was called once'); + expect(suiteResetStub, 'was called once'); + }); + + it('should forward reset to all hooks', function() { + this.suite.beforeEach(function() {}); + this.suite.afterEach(function() {}); + this.suite.beforeAll(function() {}); + this.suite.afterAll(function() {}); + sinon.stub(this.suite.getHooks('beforeEach')[0], 'reset'); + sinon.stub(this.suite.getHooks('afterEach')[0], 'reset'); + sinon.stub(this.suite.getHooks('beforeAll')[0], 'reset'); + sinon.stub(this.suite.getHooks('afterAll')[0], 'reset'); + + this.suite.reset(); + + expect(this.suite.getHooks('beforeEach')[0].reset, 'was called once'); + expect(this.suite.getHooks('afterEach')[0].reset, 'was called once'); + expect(this.suite.getHooks('beforeAll')[0].reset, 'was called once'); + expect(this.suite.getHooks('afterAll')[0].reset, 'was called once'); + }); + }); + describe('.timeout()', function() { beforeEach(function() { this.suite = new Suite('A Suite'); diff --git a/test/unit/test.spec.js b/test/unit/test.spec.js index 4ccd891bac..62a6d0667c 100644 --- a/test/unit/test.spec.js +++ b/test/unit/test.spec.js @@ -1,10 +1,24 @@ 'use strict'; +var sinon = require('sinon'); var mocha = require('../../lib/mocha'); var Test = mocha.Test; -var sinon = require('sinon'); +var Runnable = mocha.Runnable; describe('Test', function() { + /** + * @type {sinon.SinonSandbox} + */ + var sandbox; + + beforeEach(function() { + sandbox = sinon.createSandbox(); + }); + + afterEach(function() { + sandbox.restore(); + }); + describe('.clone()', function() { beforeEach(function() { this._test = new Test('To be cloned', function() {}); @@ -56,6 +70,24 @@ describe('Test', function() { }); }); + describe('.reset()', function() { + beforeEach(function() { + this._test = new Test('Test to be reset', function() {}); + }); + + it('should reset the run state', function() { + this._test.pending = true; + this._test.reset(); + expect(this._test.pending, 'to be', false); + }); + + it('should call Runnable.reset', function() { + var runnableResetStub = sandbox.stub(Runnable.prototype, 'reset'); + this._test.reset(); + expect(runnableResetStub, 'was called once'); + }); + }); + describe('.isPending()', function() { beforeEach(function() { this._test = new Test('Is it skipped', function() {});