diff --git a/docs/index.md b/docs/index.md index aeaa1c318c..ef4e4f9764 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,7 +22,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in - [test coverage reporting](#wallabyjs) - [string diff support](#diffs) - [javascript API for running tests](#more-information) -- proper exit status for CI support etc - [auto-detects and disables coloring for non-ttys](#reporters) - [async test timeout support](#delayed-root-suite) - [test retry support](#retry-tests) @@ -36,7 +35,6 @@ Mocha is a feature-rich JavaScript test framework running on [Node.js][] and in - [auto-exit to prevent "hanging" with an active loop](#-exit) - [easily meta-generate suites](#markdown) & [test-cases](#list) - [config file support](#-config-path) -- clickable suite titles to filter test execution - [node debugger support](#-inspect-inspect-brk-inspect) - [node native ES modules support](#nodejs-native-esm-support) - [detects multiple calls to `done()`](#detects-multiple-calls-to-done) @@ -449,9 +447,43 @@ setTimeout(function() { }, 5000); ``` +### Failing Hooks + +Upon a failing `before` hook all tests in the current suite and also its nested suites will be skipped. Skipped tests are included in the test results and marked as **skipped**. A skipped test is not considered a failed test. + +```js +describe('outer', function() { + before(function() { + throw new Error('Exception in before hook'); + }); + + it('should skip this outer test', function() { + // will be skipped and listed as 'skipped' + }); + + after(function() { + // will be executed + }); + + describe('inner', function() { + before(function() { + // will be skipped + }); + + it('should skip this inner test', function() { + // will be skipped and listed as 'skipped' + }); + + after(function() { + // will be skipped + }); + }); +}); +``` + ## Pending Tests -"Pending"--as in "someone should write these test cases eventually"--test-cases are simply those _without_ a callback: +"Pending" - as in "someone should write these test cases eventually" - test-cases are simply those _without_ a callback: ```js describe('Array', function() { @@ -462,7 +494,9 @@ describe('Array', function() { }); ``` -Pending tests will be included in the test results, and marked as pending. A pending test is not considered a failed test. +By appending `.skip()`, you may also tell Mocha to ignore a test: [inclusive tests](#inclusive-tests). + +Pending tests will be included in the test results, and marked as **pending**. A pending test is not considered a failed test. ## Exclusive Tests @@ -1641,7 +1675,7 @@ mocha.setup({ ### Browser-specific Option(s) Browser Mocha supports many, but not all [cli options](#command-line-usage). -To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (eg. `check-leaks` to `checkLeaks`). +To use a [cli option](#command-line-usage) that contains a "-", please convert the option to camel-case, (e.g. `check-leaks` to `checkLeaks`). #### Options that differ slightly from [cli options](#command-line-usage): diff --git a/lib/reporters/base.js b/lib/reporters/base.js index ea259445e3..7e2c20a8ca 100644 --- a/lib/reporters/base.js +++ b/lib/reporters/base.js @@ -343,18 +343,25 @@ Base.prototype.epilogue = function() { // passes fmt = color('bright pass', ' ') + - color('green', ' %d passing') + + color('green', ' %d / %d passing') + color('light', ' (%s)'); - Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration)); + Base.consoleLog(fmt, stats.passes, stats.tests, milliseconds(stats.duration)); // pending if (stats.pending) { - fmt = color('pending', ' ') + color('pending', ' %d pending'); + fmt = color('pending', ' %d pending'); Base.consoleLog(fmt, stats.pending); } + // skipped + if (stats.skipped) { + fmt = color('fail', ' %d skipped'); + + Base.consoleLog(fmt, stats.skipped); + } + // failures if (stats.failures) { fmt = color('fail', ' %d failing'); diff --git a/lib/reporters/json.js b/lib/reporters/json.js index 12b6289cd7..2540fd4d2e 100644 --- a/lib/reporters/json.js +++ b/lib/reporters/json.js @@ -9,10 +9,11 @@ var Base = require('./base'); var constants = require('../runner').constants; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; /** * Expose `JSON`. @@ -35,9 +36,10 @@ function JSONReporter(runner, options) { var self = this; var tests = []; + var passes = []; var pending = []; + var skipped = []; var failures = []; - var passes = []; runner.on(EVENT_TEST_END, function(test) { tests.push(test); @@ -55,13 +57,18 @@ function JSONReporter(runner, options) { pending.push(test); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + skipped.push(test); + }); + runner.once(EVENT_RUN_END, function() { var obj = { stats: self.stats, tests: tests.map(clean), + passes: passes.map(clean), pending: pending.map(clean), - failures: failures.map(clean), - passes: passes.map(clean) + skipped: skipped.map(clean), + failures: failures.map(clean) }; runner.testResults = obj; diff --git a/lib/reporters/spec.js b/lib/reporters/spec.js index e51ed80ac4..1983387df8 100644 --- a/lib/reporters/spec.js +++ b/lib/reporters/spec.js @@ -15,6 +15,7 @@ var EVENT_SUITE_END = constants.EVENT_SUITE_END; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var inherits = require('../utils').inherits; var color = Base.color; @@ -88,6 +89,11 @@ function Spec(runner, options) { Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title); }); + runner.on(EVENT_TEST_SKIPPED, function(test) { + var fmt = indent() + color('fail', ' - %s'); + Base.consoleLog(fmt, test.title); + }); + runner.once(EVENT_RUN_END, self.epilogue.bind(self)); } diff --git a/lib/runnable.js b/lib/runnable.js index bdd6fffe5c..0ce8b9882e 100644 --- a/lib/runnable.js +++ b/lib/runnable.js @@ -40,6 +40,7 @@ function Runnable(title, fn) { this._retries = -1; this._currentRetry = 0; this.pending = false; + this.skipped = false; } /** @@ -166,6 +167,17 @@ Runnable.prototype.isPassed = function() { return !this.isPending() && this.state === constants.STATE_PASSED; }; +/** + * Return `true` if this Runnable has been skipped. + * @return {boolean} + * @private + */ +Runnable.prototype.isSkipped = function() { + return ( + !this.pending && (this.skipped || (this.parent && this.parent.isSkipped())) + ); +}; + /** * Set or get number of retries. * @@ -270,7 +282,7 @@ Runnable.prototype.run = function(fn) { var finished; var emitted; - if (this.isPending()) return fn(); + if (this.isSkipped() || this.isPending()) return fn(); // Sometimes the ctx exists, but it is not runnable if (ctx && ctx.runnable) { @@ -458,7 +470,11 @@ var constants = utils.defineConstants( /** * Value of `state` prop when a `Runnable` has been skipped by user */ - STATE_PENDING: 'pending' + STATE_PENDING: 'pending', + /** + * Value of `state` prop when a `Runnable` has been skipped by failing hook + */ + STATE_SKIPPED: 'skipped' } ); diff --git a/lib/runner.js b/lib/runner.js index c60e562a81..0ef8b6fdba 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -19,6 +19,7 @@ var EVENT_ROOT_SUITE_RUN = Suite.constants.EVENT_ROOT_SUITE_RUN; var STATE_FAILED = Runnable.constants.STATE_FAILED; var STATE_PASSED = Runnable.constants.STATE_PASSED; var STATE_PENDING = Runnable.constants.STATE_PENDING; +var STATE_SKIPPED = Runnable.constants.STATE_SKIPPED; var dQuote = utils.dQuote; var sQuote = utils.sQuote; var stackFilter = utils.stackTraceFilter(); @@ -106,6 +107,10 @@ var constants = utils.defineConstants( * Emitted when {@link Test} becomes pending */ EVENT_TEST_PENDING: 'pending', + /** + * Emitted when {@link Test} becomes skipped + */ + EVENT_TEST_SKIPPED: 'skipped', /** * Emitted when {@link Test} execution has failed, but will retry */ @@ -358,7 +363,7 @@ Runner.prototype.hook = function(name, fn) { var hooks = suite.getHooks(name); var self = this; - function next(i) { + function nextHook(i) { var hook = hooks[i]; if (!hook) { return fn(); @@ -385,9 +390,8 @@ Runner.prototype.hook = function(name, fn) { hook.run(function cbHookRun(err) { var testError = hook.error(); - if (testError) { - self.fail(self.test, testError); - } + if (testError) self.fail(self.test, testError); + // conditional skip if (hook.pending) { if (name === HOOK_TYPE_AFTER_EACH) { @@ -418,17 +422,28 @@ Runner.prototype.hook = function(name, fn) { } } else if (err) { self.failHook(hook, err); - // stop executing hooks, notify callee of hook err - return fn(err); + + if (name === HOOK_TYPE_BEFORE_ALL) { + suite.tests.forEach(function(test) { + test.skipped = true; + }); + suite.suites.forEach(function(suite) { + suite.skipped = true; + }); + hooks = []; + } else { + // stop executing hooks, notify callee of hook err + return fn(err); + } } self.emit(constants.EVENT_HOOK_END, hook); delete hook.ctx.currentTest; - next(++i); + nextHook(++i); }); } Runner.immediately(function() { - next(0); + nextHook(0); }); }; @@ -557,15 +572,14 @@ Runner.prototype.runTests = function(suite, fn) { var test; function hookErr(_, errSuite, after) { - // before/after Each hook for errSuite failed: + // before-/afterEach hook for errSuite failed var orig = self.suite; - // for failed 'after each' hook start from errSuite parent, + // for failed afterEach hook start from errSuite parent, // otherwise start from errSuite itself self.suite = after ? errSuite.parent : errSuite; if (self.suite) { - // call hookUp afterEach self.hookUp(HOOK_TYPE_AFTER_EACH, function(err2, errSuite2) { self.suite = orig; // some hooks may fail even now @@ -576,33 +590,25 @@ Runner.prototype.runTests = function(suite, fn) { fn(errSuite); }); } else { - // there is no need calling other 'after each' hooks + // there is no need calling other afterEach hooks self.suite = orig; fn(errSuite); } } - function next(err, errSuite) { + function nextTest(err, errSuite) { // if we bail after first err - if (self.failures && suite._bail) { - tests = []; - } + if (self.failures && suite._bail) tests = []; - if (self._abort) { - return fn(); - } + if (self._abort) return fn(); - if (err) { - return hookErr(err, errSuite, true); - } + if (err) return hookErr(err, errSuite, true); // next test test = tests.shift(); // all done - if (!test) { - return fn(); - } + if (!test) return fn(); // grep var match = self._grep.test(test.fullTitle()); @@ -619,14 +625,14 @@ Runner.prototype.runTests = function(suite, fn) { // test suite don't do any immediate recursive loops. Thus, // allowing a JS runtime to breathe. if (self._grep !== self._defaultGrep) { - Runner.immediately(next); + Runner.immediately(nextTest); } else { - next(); + nextTest(); } return; } - // static skip, no hooks are executed + // static skip or conditional skip within beforeAll if (test.isPending()) { if (self.forbidPending) { self.fail(test, new Error('Pending test forbidden'), true); @@ -635,7 +641,15 @@ Runner.prototype.runTests = function(suite, fn) { self.emit(constants.EVENT_TEST_PENDING, test); } self.emit(constants.EVENT_TEST_END, test); - return next(); + return nextTest(); + } + + // skipped by failing beforeAll + if (test.isSkipped()) { + test.state = STATE_SKIPPED; + self.emit(constants.EVENT_TEST_SKIPPED, test); + self.emit(constants.EVENT_TEST_END, test); + return nextTest(); } // execute test and hook(s) @@ -655,7 +669,7 @@ Runner.prototype.runTests = function(suite, fn) { self.suite = errSuite || self.suite; return self.hookUp(HOOK_TYPE_AFTER_EACH, function(e, eSuite) { self.suite = origSuite; - next(e, eSuite); + nextTest(e, eSuite); }); } if (err) { @@ -673,7 +687,7 @@ Runner.prototype.runTests = function(suite, fn) { self.emit(constants.EVENT_TEST_PENDING, test); } self.emit(constants.EVENT_TEST_END, test); - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } else if (err) { var retry = test.currentRetry(); if (retry < test.retries()) { @@ -685,25 +699,23 @@ Runner.prototype.runTests = function(suite, fn) { // Early return + hook trigger so that it doesn't // increment the count wrong - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } else { self.fail(test, err); } self.emit(constants.EVENT_TEST_END, test); - return self.hookUp(HOOK_TYPE_AFTER_EACH, next); + return self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); } test.state = STATE_PASSED; self.emit(constants.EVENT_TEST_PASS, test); self.emit(constants.EVENT_TEST_END, test); - self.hookUp(HOOK_TYPE_AFTER_EACH, next); + self.hookUp(HOOK_TYPE_AFTER_EACH, nextTest); }); }); } - this.next = next; - this.hookErr = hookErr; - next(); + nextTest(); }; /** @@ -726,7 +738,7 @@ Runner.prototype.runSuite = function(suite, fn) { this.emit(constants.EVENT_SUITE_BEGIN, (this.suite = suite)); - function next(errSuite) { + function nextSuite(errSuite) { if (errSuite) { // current suite failed on a hook from errSuite if (errSuite === suite) { @@ -739,47 +751,37 @@ Runner.prototype.runSuite = function(suite, fn) { return done(errSuite); } - if (self._abort) { - return done(); - } + if (self._abort) return done(); var curr = suite.suites[i++]; - if (!curr) { - return done(); - } + if (!curr) return done(); // Avoid grep neglecting large number of tests causing a // huge recursive loop and thus a maximum call stack error. // See comment in `this.runTests()` for more information. if (self._grep !== self._defaultGrep) { Runner.immediately(function() { - self.runSuite(curr, next); + self.runSuite(curr, nextSuite); }); } else { - self.runSuite(curr, next); + self.runSuite(curr, nextSuite); } } function done(errSuite) { self.suite = suite; - self.nextSuite = next; // remove reference to test delete self.test; - self.hook(HOOK_TYPE_AFTER_ALL, function() { + self.hook(HOOK_TYPE_AFTER_ALL, function cbAfterAll() { self.emit(constants.EVENT_SUITE_END, suite); fn(errSuite); }); } - this.nextSuite = next; - - this.hook(HOOK_TYPE_BEFORE_ALL, function(err) { - if (err) { - return done(); - } - self.runTests(suite, next); + this.hook(HOOK_TYPE_BEFORE_ALL, function cbBeforeAll() { + self.runTests(suite, nextSuite); }); }; @@ -833,7 +835,7 @@ Runner.prototype.uncaught = function(err) { runnable.clearTimeout(); - if (runnable.isFailed()) { + if (runnable.isFailed() || runnable.isSkipped()) { // Ignore error if already failed return; } else if (runnable.isPending()) { diff --git a/lib/stats-collector.js b/lib/stats-collector.js index 938778fb0e..6b7209a959 100644 --- a/lib/stats-collector.js +++ b/lib/stats-collector.js @@ -6,13 +6,14 @@ */ var constants = require('./runner').constants; -var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; +var EVENT_TEST_PASS = constants.EVENT_TEST_PASS; +var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = constants.EVENT_TEST_SKIPPED; var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN; var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; -var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; var EVENT_RUN_END = constants.EVENT_RUN_END; -var EVENT_TEST_END = constants.EVENT_TEST_END; /** * Test statistics collector. @@ -23,6 +24,7 @@ var EVENT_TEST_END = constants.EVENT_TEST_END; * @property {number} tests - integer count of tests run. * @property {number} passes - integer count of passing tests. * @property {number} pending - integer count of pending tests. + * @property {number} skipped - integer count of skipped tests. * @property {number} failures - integer count of failed tests. * @property {Date} start - time when testing began. * @property {Date} end - time when testing concluded. @@ -47,6 +49,7 @@ function createStatsCollector(runner) { tests: 0, passes: 0, pending: 0, + skipped: 0, failures: 0 }; @@ -71,6 +74,9 @@ function createStatsCollector(runner) { runner.on(EVENT_TEST_PENDING, function() { stats.pending++; }); + runner.on(EVENT_TEST_SKIPPED, function() { + stats.skipped++; + }); runner.on(EVENT_TEST_END, function() { stats.tests++; }); diff --git a/lib/suite.js b/lib/suite.js index 191d946b50..00c47f51da 100644 --- a/lib/suite.js +++ b/lib/suite.js @@ -62,6 +62,7 @@ function Suite(title, parentContext, isRoot) { this.suites = []; this.tests = []; this.pending = false; + this.skipped = false; this._beforeEach = []; this._beforeAll = []; this._afterEach = []; @@ -210,6 +211,17 @@ Suite.prototype.isPending = function() { return this.pending || (this.parent && this.parent.isPending()); }; +/** + * Check if this suite or its parent suite is marked as skipped. + * + * @private + */ +Suite.prototype.isSkipped = function() { + return ( + !this.pending && (this.skipped || (this.parent && this.parent.isSkipped())) + ); +}; + /** * Generic hook-creator. * @private diff --git a/test/assertions.js b/test/assertions.js index 7453392059..f3d1018bfd 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -35,6 +35,7 @@ exports.mixinMochaAssertions = function(expect) { typeof v.passing === 'number' && typeof v.failing === 'number' && typeof v.pending === 'number' && + typeof v.skipped === 'number' && typeof v.output === 'string' && typeof v.code === 'number' ); @@ -143,6 +144,12 @@ exports.mixinMochaAssertions = function(expect) { expect(result.stats.pending, '[not] to be', count); } ) + .addAssertion( + ' [not] to have skipped [test] count ', + function(expect, result, count) { + expect(result.stats.skipped, '[not] to be', count); + } + ) .addAssertion( ' [not] to have run (test|tests) ', function(expect, result, titles) { @@ -266,12 +273,23 @@ exports.mixinMochaAssertions = function(expect) { ); } ) - .addAssertion(' [not] to have pending tests', function( - expect, - result - ) { - expect(result.stats.pending, '[not] to be greater than', 0); - }) + .addAssertion( + ' [not] to have skipped test order ', + function(expect, result, titles) { + expect(result, '[not] to have test order', 'skipped', titles); + } + ) + .addAssertion( + ' [not] to have skipped test order ', + function(expect, result, titles) { + expect( + result, + '[not] to have test order', + 'skipped', + Array.prototype.slice.call(arguments, 2) + ); + } + ) .addAssertion(' [not] to have passed tests', function( expect, result @@ -284,6 +302,18 @@ exports.mixinMochaAssertions = function(expect) { ) { expect(result.stats.failed, '[not] to be greater than', 0); }) + .addAssertion(' [not] to have pending tests', function( + expect, + result + ) { + expect(result.stats.pending, '[not] to be greater than', 0); + }) + .addAssertion(' [not] to have skipped tests', function( + expect, + result + ) { + expect(result.stats.skipped, '[not] to be greater than', 0); + }) .addAssertion(' [not] to have tests', function( expect, result diff --git a/test/integration/fixtures/glob/glob.spec.js b/test/integration/fixtures/glob/glob.spec.js index c6c3d26a69..e5e5a80f0d 100644 --- a/test/integration/fixtures/glob/glob.spec.js +++ b/test/integration/fixtures/glob/glob.spec.js @@ -1,7 +1,7 @@ 'use strict'; describe('globbing test', function() { - it('should find this test', function() { + it('should find this glob/test', function() { // see test/integration/glob.spec.js for details }); }); diff --git a/test/integration/fixtures/glob/nested/glob.spec.js b/test/integration/fixtures/glob/nested/glob.spec.js index c6c3d26a69..0bdaee1484 100644 --- a/test/integration/fixtures/glob/nested/glob.spec.js +++ b/test/integration/fixtures/glob/nested/glob.spec.js @@ -1,7 +1,7 @@ 'use strict'; describe('globbing test', function() { - it('should find this test', function() { + it('should find this glob/nested/test', function() { // see test/integration/glob.spec.js for details }); }); diff --git a/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js b/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js index 04801c1946..8e69274102 100644 --- a/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-async-error-tip.fixture.js @@ -1,7 +1,7 @@ 'use strict'; describe('spec 1', function () { - it('should not blame me', function () { }); + it('should run test-1', function () { }); }); describe('spec 2', function () { before(function (done) { @@ -9,5 +9,5 @@ describe('spec 2', function () { throw new Error('before hook error'); }); }); - it('skipped'); + it('should not run test-2', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-async-error.fixture.js b/test/integration/fixtures/hooks/before-hook-async-error.fixture.js index 2530eec783..52cc7139cc 100644 --- a/test/integration/fixtures/hooks/before-hook-async-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-async-error.fixture.js @@ -2,20 +2,13 @@ describe('spec 1', function () { before(function (done) { - console.log('before'); process.nextTick(function () { throw new Error('before hook error'); }); }); - it('should not be called because of error in before hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before hook', function () { - console.log('test 2'); - }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); }); describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); + it('should run test-3', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js b/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js index 804e5e415b..d456464bd7 100644 --- a/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-deepnested-error.fixture.js @@ -1,13 +1,13 @@ 'use strict'; describe('spec 1', function () { - it('should pass', function () { }); - describe('spec 2 nested - this title should be used', function () { + it('should run test-1', function () { }); + describe('nested spec 2', function () { before(function() { throw new Error('before hook nested error'); }); - describe('spec 3 nested', function () { - it('it nested - this title should not be used', function () { }); + describe('deepnested spec 3', function () { + it('should not run deepnested test-2', function () { }); }); }); }); diff --git a/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js b/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js index 64df731573..d439701e5c 100644 --- a/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-error-tip.fixture.js @@ -1,11 +1,11 @@ 'use strict'; describe('spec 1', function () { - it('should not blame me', function () { }); + it('should run test-1', function () { }); }); describe('spec 2', function () { before(function () { throw new Error('before hook error'); }); - it('skipped'); + it('should not run test-2', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-error.fixture.js b/test/integration/fixtures/hooks/before-hook-error.fixture.js index 547e54a243..8263d5c855 100644 --- a/test/integration/fixtures/hooks/before-hook-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-error.fixture.js @@ -2,18 +2,11 @@ describe('spec 1', function () { before(function () { - console.log('before'); throw new Error('before hook error'); }); - it('should not be called because of error in before hook', function () { - console.log('test 1'); - }); - it('should not be called because of error in before hook', function () { - console.log('test 2'); - }); + it('should not run test-1', function () { }); + it('should not run test-2', function () { }); }); describe('spec 2', function () { - it('should be called, because hook error was in a sibling suite', function () { - console.log('test 3'); - }); + it('should run test-3', function () { }); }); diff --git a/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js b/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js index c0ade3a9f4..ea06c31c81 100644 --- a/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js +++ b/test/integration/fixtures/hooks/before-hook-nested-error.fixture.js @@ -1,11 +1,11 @@ 'use strict'; describe('spec 1', function () { - it('should pass', function () { }); + it('should run test-1', function () { }); describe('spec nested', function () { before(function() { throw new Error('before hook nested error'); }); - it('it nested - this title should be used', function () { }); + it('should not run nested test-2', function () { }); }); }); diff --git a/test/integration/glob.spec.js b/test/integration/glob.spec.js index 4284320aa7..c6ce0077f8 100644 --- a/test/integration/glob.spec.js +++ b/test/integration/glob.spec.js @@ -11,10 +11,9 @@ describe('globbing', function() { testGlob.shouldSucceed( './*.js', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); }, done @@ -39,8 +38,11 @@ describe('globbing', function() { testGlob.shouldFail( './*-none.js ./*-none-twice.js', function(results) { - expect(results.stderr, 'to contain', 'Error: No test files found'); - expect(results.stderr, 'not to contain', '*-none'); + expect( + results.stderr, + 'to contain', + 'Error: No test files found' + ).and('not to contain', '*-none'); }, done ); @@ -50,10 +52,9 @@ describe('globbing', function() { testGlob.shouldSucceed( './*.js ./*-none.js', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); expect( results.stderr, @@ -71,10 +72,9 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./*.js"', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); }, done @@ -109,10 +109,9 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./*.js" "./*-none.js"', function(results) { - expect( - results.stdout, + expect(results.stdout, 'to contain', '["start",{"total":1}]').and( 'to contain', - '["end",{"suites":1,"tests":1,"passes":1,"pending":0,"failures":0,' + '["pass",{"title":"should find this glob/test"' ); expect( results.stderr, @@ -129,11 +128,15 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./**/*.js"', function(results) { - expect( - results.stdout, - 'to contain', - '["end",{"suites":2,"tests":2,"passes":2,"pending":0,"failures":0,' - ); + expect(results.stdout, 'to contain', '["start",{"total":2}]') + .and( + 'to contain', + '["pass",{"title":"should find this glob/test"' + ) + .and( + 'to contain', + '["pass",{"title":"should find this glob/nested/test"' + ); }, done ); @@ -157,11 +160,15 @@ describe('globbing', function() { testGlob.shouldSucceed( '"./**/*.js" "./**/*-none.js"', function(results) { - expect( - results.stdout, - 'to contain', - '["end",{"suites":2,"tests":2,"passes":2,"pending":0,"failures":0,' - ); + expect(results.stdout, 'to contain', '["start",{"total":2}]') + .and( + 'to contain', + '["pass",{"title":"should find this glob/test"' + ) + .and( + 'to contain', + '["pass",{"title":"should find this glob/nested/test"' + ); expect( results.stderr, 'to contain', @@ -187,13 +194,6 @@ var testGlob = { }) }; -var isFlakeyNode = (function() { - var version = process.versions.node.split('.'); - return ( - version[0] === '0' && version[1] === '10' && process.platform === 'win32' - ); -})(); - function execMochaWith(validate) { return function execMocha(glob, assertOn, done) { exec( @@ -206,12 +206,8 @@ function execMochaWith(validate) { function(error, stdout, stderr) { try { validate(error, stderr); - if (isFlakeyNode && error && stderr === '') { - execMocha(glob, assertOn, done); - } else { - assertOn({stdout: stdout, stderr: stderr}); - done(); - } + assertOn({stdout: stdout, stderr: stderr}); + done(); } catch (assertion) { done(assertion); } diff --git a/test/integration/helpers.js b/test/integration/helpers.js index 6cdf7e93cf..0465ba5671 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -15,15 +15,16 @@ module.exports = { * Invokes the mocha binary for the given fixture with color output disabled. * Accepts an array of additional command line args to pass. The callback is * invoked with a summary of the run, in addition to its output. The summary - * includes the number of passing, pending, and failing tests, as well as the - * exit code. Useful for testing different reporters. + * includes the number of passing, pending, skipped and failing tests, as well + * as the exit code. Useful for testing different reporters. * * By default, `STDERR` is ignored. Pass `{stdio: 'pipe'}` as `opts` if you * want it. * Example response: * { - * pending: 0, * passing: 0, + * pending: 0, + * skipped: 0, * failing: 1, * code: 1, * output: '...' @@ -306,13 +307,17 @@ function resolveFixturePath(fixture) { } function getSummary(res) { - return ['passing', 'pending', 'failing'].reduce(function(summary, type) { + return ['passing', 'pending', 'skipped', 'failing'].reduce(function( + summary, + type + ) { var pattern, match; - pattern = new RegExp(' (\\d+) ' + type + '\\s'); + pattern = new RegExp(' (\\d+) (?:/ \\d+ )?' + type + '\\s'); match = pattern.exec(res.output); summary[type] = match ? parseInt(match, 10) : 0; return summary; - }, res); + }, + res); } diff --git a/test/integration/hook-err.spec.js b/test/integration/hook-err.spec.js index d5fe6e858d..19c4cec727 100644 --- a/test/integration/hook-err.spec.js +++ b/test/integration/hook-err.spec.js @@ -9,72 +9,161 @@ var bang = require('../../lib/reporters/base').symbols.bang; describe('hook error handling', function() { var lines; - describe('before hook error', function() { - before(run('hooks/before-hook-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); - }); - }); + describe('synchronous hook', function() { + describe('in before', function() { + it('before hook error', function(done) { + var fixture = 'hooks/before-hook-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before all" hook for "should not run test-1"' + ); + done(); + }); + }); - describe('before hook error tip', function() { - before(run('hooks/before-hook-error-tip.fixture.js', onlyErrorTitle())); - it('should verify results', function() { - expect(lines, 'to equal', [ - '1) spec 2', - '"before all" hook for "skipped":' - ]); - }); - }); + it('before hook error tip', function(done) { + var fixture = 'hooks/before-hook-error-tip.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run test-2"' + ); + done(); + }); + }); - describe('before hook root error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-root-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook root error') - .and('to have failed test', '"before all" hook in "{root}"') - .and('to have passed test count', 0); - done(); + it('before hook root error', function(done) { + var fixture = 'hooks/before-hook-root-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook root error') + .and('to have test count', 1) + .and('to have passed test count', 0) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have skipped test order', 'should not be called') + .and('to have failed test', '"before all" hook in "{root}"'); + done(); + }); }); - }); - }); - describe('before hook nested error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-nested-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook nested error') - .and( - 'to have failed test', - '"before all" hook for "it nested - this title should be used"' - ) - .and('to have passed test count', 1) - .and('to have passed test', 'should pass'); - done(); + it('before hook nested error', function(done) { + var fixture = 'hooks/before-hook-nested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook nested error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run nested test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run nested test-2"' + ); + done(); + }); + }); + + it('before hook deepnested error', function(done) { + var fixture = 'hooks/before-hook-deepnested-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook nested error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and( + 'to have skipped test order', + 'should not run deepnested test-2' + ) + .and('to have failed test', '"before all" hook in "nested spec 2"'); + done(); + }); }); }); }); - describe('before hook deepnested error', function() { - it('should verify results', function(done) { - var fixture = 'hooks/before-hook-deepnested-error.fixture.js'; - runMochaJSON(fixture, [], function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed with error', 'before hook nested error') - .and( - 'to have failed test', - '"before all" hook in "spec 2 nested - this title should be used"' - ) - .and('to have passed test count', 1) - .and('to have passed test', 'should pass'); - done(); + describe('asynchronous hook', function() { + describe('in before', function() { + it('before hook error', function(done) { + var fixture = 'hooks/before-hook-async-error.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 3) + .and('to have passed test count', 1) + .and('to have skipped test count', 2) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-3') + .and( + 'to have skipped test order', + 'should not run test-1', + 'should not run test-2' + ) + .and( + 'to have failed test', + '"before all" hook for "should not run test-1"' + ); + done(); + }); + }); + + it('before hook error tip', function(done) { + var fixture = 'hooks/before-hook-async-error-tip.fixture.js'; + runMochaJSON(fixture, [], function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', 'before hook error') + .and('to have test count', 2) + .and('to have passed test count', 1) + .and('to have skipped test count', 1) + .and('to have failed test count', 1) + .and('to have passed test', 'should run test-1') + .and('to have skipped test order', 'should not run test-2') + .and( + 'to have failed test', + '"before all" hook for "should not run test-2"' + ); + done(); + }); }); }); }); @@ -182,25 +271,6 @@ describe('hook error handling', function() { }); }); - describe('async - before hook error', function() { - before(run('hooks/before-hook-async-error.fixture.js')); - it('should verify results', function() { - expect(lines, 'to equal', ['before', bang + 'test 3']); - }); - }); - - describe('async - before hook error tip', function() { - before( - run('hooks/before-hook-async-error-tip.fixture.js', onlyErrorTitle()) - ); - it('should verify results', function() { - expect(lines, 'to equal', [ - '1) spec 2', - '"before all" hook for "skipped":' - ]); - }); - }); - describe('async - before each hook error', function() { before(run('hooks/beforeEach-hook-async-error.fixture.js')); it('should verify results', function() { @@ -284,17 +354,3 @@ function onlyConsoleOutput() { return !foundSummary && line.length > 0; }; } - -function onlyErrorTitle(line) { - var foundErrorTitle = false; - var foundError = false; - return function(line) { - if (!foundErrorTitle) { - foundErrorTitle = !!/^1\)/.exec(line); - } - if (!foundError) { - foundError = /Error:/.exec(line); - } - return foundErrorTitle && !foundError; - }; -} diff --git a/test/reporters/base.spec.js b/test/reporters/base.spec.js index 744b92e69b..8e81223eab 100644 --- a/test/reporters/base.spec.js +++ b/test/reporters/base.spec.js @@ -426,25 +426,43 @@ describe('Base reporter', function() { }); describe('when reporter output immune to user test changes', function() { - var sandbox; var baseConsoleLog; beforeEach(function() { - sandbox = sinon.createSandbox(); sandbox.stub(console, 'log'); baseConsoleLog = sandbox.stub(Base, 'consoleLog'); }); it('should let you stub out console.log without effecting reporters output', function() { Base.list([]); - baseConsoleLog.restore(); expect(baseConsoleLog, 'was called'); expect(console.log, 'was not called'); + sandbox.restore(); }); + }); + + describe('epilogue', function() { + it('should include "pending" count', function() { + var ctx = {stats: {passes: 0, pending: 2, skipped: 0, duration: 12}}; + var epilogue = Base.prototype.epilogue.bind(ctx); - afterEach(function() { + epilogue(); sandbox.restore(); + + var out = stdout.join('\n').trim(); + expect(out, 'to contain', '2 pending').and('not to contain', 'skipped'); + }); + + it('should include "skipped" count', function() { + var ctx = {stats: {passes: 0, pending: 0, skipped: 3, duration: 12}}; + var epilogue = Base.prototype.epilogue.bind(ctx); + + epilogue(); + sandbox.restore(); + + var out = stdout.join('\n').trim(); + expect(out, 'to contain', '3 skipped').and('not to contain', 'pending'); }); }); }); diff --git a/test/reporters/helpers.js b/test/reporters/helpers.js index 45c4d916de..90567df37e 100644 --- a/test/reporters/helpers.js +++ b/test/reporters/helpers.js @@ -60,6 +60,7 @@ function createRunnerFunction(runStr, ifStr1, ifStr2, ifStr3, arg1, arg2) { } }; case 'pending test': + case 'skipped test': case 'pass': case 'fail': case 'suite': diff --git a/test/reporters/json.spec.js b/test/reporters/json.spec.js index f6299dd134..142634fd57 100644 --- a/test/reporters/json.spec.js +++ b/test/reporters/json.spec.js @@ -80,6 +80,26 @@ describe('JSON reporter', function() { }); }); + it('should have 1 test skipped', function(done) { + suite.skipped = true; + suite.addTest(new Test(testTitle, noop)); + + runner.run(function(failureCount) { + sandbox.restore(); + expect(runner, 'to satisfy', { + testResults: { + skipped: [ + { + title: testTitle + } + ] + } + }); + expect(failureCount, 'to be', 0); + done(); + }); + }); + it('should handle circular objects in errors', function(done) { var testTitle = 'json test 1'; function CircleError() { diff --git a/test/reporters/spec.spec.js b/test/reporters/spec.spec.js index 608bc7f512..082d5a203e 100644 --- a/test/reporters/spec.spec.js +++ b/test/reporters/spec.spec.js @@ -14,6 +14,7 @@ var EVENT_SUITE_BEGIN = events.EVENT_SUITE_BEGIN; var EVENT_TEST_FAIL = events.EVENT_TEST_FAIL; var EVENT_TEST_PASS = events.EVENT_TEST_PASS; var EVENT_TEST_PENDING = events.EVENT_TEST_PENDING; +var EVENT_TEST_SKIPPED = events.EVENT_TEST_SKIPPED; describe('Spec reporter', function() { var runReporter = makeRunReporter(Spec); @@ -73,6 +74,27 @@ describe('Spec reporter', function() { }); }); + describe("on 'skipped' event", function() { + it('should return title', function() { + var suite = { + title: expectedTitle + }; + var runner = createMockRunner( + 'skipped test', + EVENT_TEST_SKIPPED, + null, + null, + suite + ); + var options = {}; + var stdout = runReporter({epilogue: noop}, runner, options); + sandbox.restore(); + + var expectedArray = [' - ' + expectedTitle + '\n']; + expect(stdout, 'to equal', expectedArray); + }); + }); + describe("on 'pass' event", function() { describe('when test speed is slow', function() { it('should return expected tick, title, and duration', function() {