diff --git a/lib/cli.js b/lib/cli.js index 1d71df41a..56c0eb9b6 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -46,6 +46,13 @@ function processArgs (argv, options, fs, path) { options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true' } + if (helper.isString(options.failOnFailingTestSuite)) { + options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true' + } else if (typeof options.failOnFailingTestSuite === 'undefined'){ + // Default value to true, to avoid chanching application default behaviour + options.failOnFailingTestSuite = true + } + if (helper.isString(options.formatError)) { let required try { @@ -196,6 +203,8 @@ function describeStart () { .describe('report-slower-than', ' Report tests that are slower than given time [ms].') .describe('fail-on-empty-test-suite', 'Fail on empty test suite.') .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.') + .describe('fail-on-failing-test-suite', 'Fail on failing test suite.') + .describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.') .describe('help', 'Print usage and options.') } diff --git a/lib/server.js b/lib/server.js index 043c8a29c..c81c105df 100644 --- a/lib/server.js +++ b/lib/server.js @@ -133,6 +133,18 @@ class Server extends KarmaEventEmitter { return this._fileList ? this._fileList.refresh() : Promise.resolve() } + calculateExitCode (results, config) { + if (results.success + results.failed === 0) { + if (!config.failOnEmptyTestSuite) { + this.log.warn('Test suite was empty.') + return 0 + } + } else if (!results.error && !results.disconnected && !config.failOnFailingTestSuite) { + return 0 + } + return results.exitCode + } + _start (config, launcher, preprocess, fileList, capturedBrowsers, executor, done) { if (config.detached) { this._detach(config, done) @@ -250,9 +262,8 @@ class Server extends KarmaEventEmitter { const results = singleRunBrowsers.getResults() if (singleRunBrowserNotCaptured) { results.exitCode = 1 - } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) { - results.exitCode = 0 - this.log.warn('Test suite was empty.') + } else { + results.exitCode = this.calculateExitCode(results, config) } this.emit('run_complete', singleRunBrowsers, results) } diff --git a/test/unit/server.spec.js b/test/unit/server.spec.js index fcd2f9b61..eb597773d 100644 --- a/test/unit/server.spec.js +++ b/test/unit/server.spec.js @@ -258,4 +258,163 @@ describe('server', () => { } }) }) + + // ============================================================================ + // server.calculateExitCode + // ============================================================================ + describe('calculateExitCode', () => { + const DEFAULT_EXIT_CODE = -1 + const EXIT_CODE_SUCCESS = 0 + const DEFAULT_OPTIONS = { + failOnEmptyTestSuite: false, + failOnFailingTestSuite: true + } + + describe('no tests, with error', () => { + const results = { + success: 0, + failed: 0, + error: true, + exitCode: DEFAULT_EXIT_CODE + } + it('shall be unmodified if failOnEmptyTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall pass if failOnEmptyTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: false + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnFailingTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: true + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnFailingTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: false + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + }) + describe('no tests, no errors', () => { + const results = { + success: 0, + failed: 0, + error: false, + exitCode: DEFAULT_EXIT_CODE + } + it('shall pass if failOnEmptyTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: false + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall be unmodifed if failOnEmptyTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall pass if failOnFailingTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true, + failOnFailingTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodifed if failOnFailingTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true, + failOnFailingTestSuite: false + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + }) + describe('all test passed, no errors', () => { + const results = { + success: 10, + failed: 0, + error: false, + exitCode: DEFAULT_EXIT_CODE + } + it('shall be unmodifed if failOnEmptyTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodifed if failOnEmptyTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodifed if failOnFailingTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodifed if failOnFailingTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: false + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + }) + describe('all test passed, with error', () => { + const results = { + success: 10, + failed: 5, + error: false, + exitCode: DEFAULT_EXIT_CODE + } + it('shall be unmodified if failOnEmptyTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodified if failOnEmptyTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnEmptyTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall be unmodified if failOnFailingTestSuite is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: true + }) + expect(res).to.be.equal(DEFAULT_EXIT_CODE) + }) + it('shall pass if failOnFailingTestSuite not is set', () => { + const res = server.calculateExitCode(results, { + ...DEFAULT_OPTIONS, + failOnFailingTestSuite: false + }) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + }) + }) })