Skip to content

Commit

Permalink
fix(runner): Make exit code configurable when tests are failing
Browse files Browse the repository at this point in the history 
Fixes #1300
  • Loading branch information
@andreaspsson
andreaspsson committed Aug 23, 2018
1 parent eeadcf2 commit dc684c6
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 3 deletions.
9 changes: 9 additions & 0 deletions lib/cli.js
View file
Expand Up @@ -46,6 +46,13 @@ function processArgs (argv, options, fs, path) {
options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
}

if (helper.isString(options.failOnFailingTestSuite)) {
options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
} else if (typeof options.failOnFailingTestSuite === 'undefined'){
// Default value to true, to avoid chanching application default behaviour
options.failOnFailingTestSuite = true
}

if (helper.isString(options.formatError)) {
let required
try {
Expand Down Expand Up @@ -196,6 +203,8 @@ function describeStart () {
.describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
.describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
.describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
.describe('help', 'Print usage and options.')
}

Expand Down
17 changes: 14 additions & 3 deletions lib/server.js
View file
Expand Up @@ -133,6 +133,18 @@ class Server extends KarmaEventEmitter {
return this._fileList ? this._fileList.refresh() : Promise.resolve()
}

calculateExitCode (results, config) {
if (results.success + results.failed === 0) {
if (!config.failOnEmptyTestSuite) {
this.log.warn('Test suite was empty.')
return 0
}
} else if (!results.error && !results.disconnected && !config.failOnFailingTestSuite) {
return 0
}
return results.exitCode
}

_start (config, launcher, preprocess, fileList, capturedBrowsers, executor, done) {
if (config.detached) {
this._detach(config, done)
Expand Down Expand Up @@ -250,9 +262,8 @@ class Server extends KarmaEventEmitter {
const results = singleRunBrowsers.getResults()
if (singleRunBrowserNotCaptured) {
results.exitCode = 1
} else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
results.exitCode = 0
this.log.warn('Test suite was empty.')
} else {
results.exitCode = this.calculateExitCode(results, config)
}
this.emit('run_complete', singleRunBrowsers, results)
}
Expand Down
159 changes: 159 additions & 0 deletions test/unit/server.spec.js
View file
Expand Up @@ -258,4 +258,163 @@ describe('server', () => {
}
})
})

// ============================================================================
// server.calculateExitCode
// ============================================================================
describe('calculateExitCode', () => {
const DEFAULT_EXIT_CODE = -1
const EXIT_CODE_SUCCESS = 0
const DEFAULT_OPTIONS = {
failOnEmptyTestSuite: false,
failOnFailingTestSuite: true
}

describe('no tests, with error', () => {
const results = {
success: 0,
failed: 0,
error: true,
exitCode: DEFAULT_EXIT_CODE
}
it('shall be unmodified if failOnEmptyTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: false
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: true
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: false
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
describe('no tests, no errors', () => {
const results = {
success: 0,
failed: 0,
error: false,
exitCode: DEFAULT_EXIT_CODE
}
it('shall pass if failOnEmptyTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: false
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall be unmodifed if failOnEmptyTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall pass if failOnFailingTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true,
failOnFailingTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodifed if failOnFailingTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true,
failOnFailingTestSuite: false
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
})
describe('all test passed, no errors', () => {
const results = {
success: 10,
failed: 0,
error: false,
exitCode: DEFAULT_EXIT_CODE
}
it('shall be unmodifed if failOnEmptyTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodifed if failOnEmptyTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodifed if failOnFailingTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodifed if failOnFailingTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: false
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
describe('all test passed, with error', () => {
const results = {
success: 10,
failed: 5,
error: false,
exitCode: DEFAULT_EXIT_CODE
}
it('shall be unmodified if failOnEmptyTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodified if failOnEmptyTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnEmptyTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall be unmodified if failOnFailingTestSuite is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: true
})
expect(res).to.be.equal(DEFAULT_EXIT_CODE)
})
it('shall pass if failOnFailingTestSuite not is set', () => {
const res = server.calculateExitCode(results, {
...DEFAULT_OPTIONS,
failOnFailingTestSuite: false
})
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
})
})

0 comments on commit dc684c6

Please sign in to comment.