Skip to content

Commit

Permalink
Tests: Make CLI test cases easier to debug with normalised stack
Browse files Browse the repository at this point in the history 
Remove use of regexes and instead strictly compare exact strings,
but normalise two things:

* collapse any Node internals into a single frame called "internal".
* strip line/column numbers of our own build artefact.
  • Loading branch information
@Krinkle
Krinkle committed Mar 13, 2021
1 parent eeaa482 commit e295106
Show file tree
Hide file tree
Showing 4 changed files with 190 additions and 186 deletions.
58 changes: 25 additions & 33 deletions test/cli/fixtures/expected/tap-outputs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,13 @@ not ok 1 Throws match > bad
severity: failed
actual : "Error: Match me with a pattern"
expected: "/incorrect pattern/"
stack: .*
stack: at Object.<anonymous> (/qunit/test/cli/fixtures/fail/throws-match.js:3:10)
...
1..1
# pass 0
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"qunit test single.js 'glob/**/*-test.js'":
`TAP version 13
Expand Down Expand Up @@ -140,10 +139,9 @@ not ok 2 global failure
# pass 0
# skip 0
# todo 0
# fail 2
`,
# fail 2`,

// Ignore the last frame about Node processing ticks (differs between Node 10 ad 12+)
// The last frame differs between Node 10 and 12+ (changes in processing of ticks)
"qunit no-tests":
`TAP version 13
not ok 1 global failure
Expand All @@ -153,18 +151,17 @@ not ok 1 global failure
actual : {}
expected: undefined
stack: Error: No tests were run.
at done (.*)
at advanceTestQueue (.*)
at Object.advance (.*)
at unblockAndAdvanceQueue (.*)(\n at <anonymous>)?
at .*
at done (/qunit/qunit/qunit.js)
at advanceTestQueue (/qunit/qunit/qunit.js)
at Object.advance (/qunit/qunit/qunit.js)
at unblockAndAdvanceQueue (/qunit/qunit/qunit.js)
at internal
...
1..1
# pass 0
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"qunit sourcemap/source.js":
`TAP version 13
Expand All @@ -181,8 +178,7 @@ not ok 2 Example > bad
# pass 1
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"NODE_OPTIONS='--enable-source-maps' qunit sourcemap/source.min.js":
`TAP version 13
Expand All @@ -193,19 +189,18 @@ not ok 2 Example > bad
severity: failed
actual : false
expected: true
stack: at .* \\(.*source.min.js:1:.*\\)
-> .*source.js:7:10
stack: at Object.<anonymous> (/qunit/test/cli/fixtures/sourcemap/source.min.js:1:133)
-> /qunit/test/cli/fixtures/sourcemap/sourcemap/source.js:7:10
...
1..2
# pass 1
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"qunit ../../es2018/esm.mjs":
`TAP version 13
ok 1 ESM test suite > sum\\(\\)
ok 1 ESM test suite > sum()
1..1
# pass 1
# skip 0
Expand All @@ -220,16 +215,14 @@ not ok 1 timeout > first
severity: failed
actual : null
expected: undefined
stack: at .* (.*timers.js.*)
at .*(\n at .*)?(\n at .*)?
stack: at internal
...
ok 2 timeout > second
1..2
# pass 1
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"qunit zero-assertions.js":
`TAP version 13
Expand All @@ -250,18 +243,17 @@ not ok 1 global failure
actual : {}
expected: undefined
stack: Error: No tests matched the filter "no matches".
at done (.*)
at advanceTestQueue (.*)
at Object.advance (.*)
at unblockAndAdvanceQueue (.*)(\n at <anonymous>)?
at .*
at done (/qunit/qunit/qunit.js)
at advanceTestQueue (/qunit/qunit/qunit.js)
at Object.advance (/qunit/qunit/qunit.js)
at unblockAndAdvanceQueue (/qunit/qunit/qunit.js)
at internal
...
1..1
# pass 0
# skip 0
# todo 0
# fail 1
`,
# fail 1`,

"qunit single.js --require require-dep --require './node_modules/require-dep/module.js'":
`required require-dep/index.js
Expand Down Expand Up @@ -302,7 +294,7 @@ not ok 1 # TODO module B > Only this module should run > a todo test
severity: todo
actual : false
expected: true
stack: .*
stack: at Object.<anonymous> (/qunit/test/cli/fixtures/only/module.js:17:15)
...
ok 2 # SKIP module B > Only this module should run > implicitly skipped test
ok 3 module B > Only this module should run > normal test
Expand All @@ -323,7 +315,7 @@ not ok 1 # TODO module B > test B
severity: todo
actual : false
expected: true
stack: .*
stack: at Object.<anonymous> (/qunit/test/cli/fixtures/only/module-flat.js:9:13)
...
ok 2 # SKIP module B > test C
ok 3 module B > test D
Expand Down
34 changes: 32 additions & 2 deletions test/cli/helpers/execute.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,48 @@

const path = require( "path" );
const exec = require( "execa" ).shell;
const reEscape = /([\\{}()|.?*+\-^$[\]])/g;

// Apply light normalization to CLI output to allow strict string
// comparison across Node versions and OS platforms against the
// expected output in fixtures/.
function normalize( actual ) {
const dir = path.join( __dirname, "..", "..", ".." );
const reDir = new RegExp( dir.replace( reEscape, "\\$1" ), "g" );

return actual
.replace( reDir, "/qunit" )
.replace( /(\/qunit\/qunit\/qunit\.js):\d+:\d+\)/g, "$1)" )
.replace( / at .+\([^/)][^)]*\)/g, " at internal" )

// merge successive lines after initial frame
.replace( /(\n\s+at internal)+/g, "$1" )

// merge successive line with initial frame
.replace( /(at internal)\n\s+at internal/g, "$1" );
}

// Executes the provided command from within the fixtures directory
// The execaOptions parameter is used by test/cli/watch.js to
// control the stdio stream.
module.exports = function execute( command, execaOptions ) {
module.exports = async function execute( command, execaOptions, hook ) {
const cwd = process.cwd();
process.chdir( path.join( __dirname, "..", "fixtures" ) );

command = command.replace( /(^| )qunit\b/, "$1../../../bin/qunit.js" );
const execution = exec( command, execaOptions );
if ( hook ) {
hook( execution );
}

process.chdir( cwd );

return execution;
try {
const result = await execution;
result.stdout = normalize( String( result.stdout ).trimEnd() );
return result;
} catch ( e ) {
e.stdout = normalize( String( e.stdout ).trimEnd() );
throw e;
}
};
39 changes: 8 additions & 31 deletions test/cli/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,6 @@ const expectedOutput = require( "./fixtures/expected/tap-outputs" );
const execute = require( "./helpers/execute" );
const semver = require( "semver" );

QUnit.assert.matches = function( actual, expected, message ) {
this.pushResult( {
result: expected.test( actual ),
actual,
expected: expected.toString(),
message
} );
};

QUnit.module( "CLI Main", function() {
QUnit.test( "defaults to running tests in 'test' directory", async function( assert ) {
const command = "qunit";
Expand Down Expand Up @@ -93,8 +84,7 @@ QUnit.module( "CLI Main", function() {
} catch ( e ) {
assert.equal( e.code, 1 );
assert.equal( e.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( e.stdout ), true );
assert.equal( e.stdout, expectedOutput[ command ] );
}
} );

Expand All @@ -113,8 +103,7 @@ QUnit.module( "CLI Main", function() {
} catch ( e ) {
assert.equal( e.code, 1 );
assert.equal( e.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( e.stdout ), true );
assert.equal( e.stdout, expectedOutput[ command ] );
}
} );

Expand Down Expand Up @@ -158,11 +147,7 @@ QUnit.module( "CLI Main", function() {
assert.equal( execution.stderr, "" );
}

const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( execution.stdout ), true );
if ( !re.test( execution.stdout ) ) {
assert.equal( execution.stdout, expectedOutput[ command ] );
}
assert.equal( execution.stdout, expectedOutput[ command ] );
} );
}

Expand All @@ -188,11 +173,7 @@ QUnit.module( "CLI Main", function() {
} catch ( e ) {
assert.equal( e.code, 1 );
assert.equal( e.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( e.stdout ), true );
if ( !re.test( e.stdout ) ) {
assert.equal( e.stdout, expectedOutput[ command ] );
}
assert.equal( e.stdout, expectedOutput[ command ] );
}
} );
}
Expand All @@ -204,8 +185,7 @@ QUnit.module( "CLI Main", function() {
} catch ( e ) {
assert.equal( e.code, 1 );
assert.equal( e.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( e.stdout ), true );
assert.equal( e.stdout, expectedOutput[ command ] );
}
} );

Expand Down Expand Up @@ -249,8 +229,7 @@ QUnit.module( "CLI Main", function() {
} catch ( e ) {
assert.equal( e.code, 1 );
assert.equal( e.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.equal( re.test( e.stdout ), true );
assert.equal( e.stdout, expectedOutput[ command ] );
}
} );
} );
Expand Down Expand Up @@ -323,8 +302,7 @@ QUnit.module( "CLI Main", function() {

assert.equal( execution.code, 0 );
assert.equal( execution.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.matches( execution.stdout, re );
assert.equal( execution.stdout, expectedOutput[ command ] );
} );

QUnit.test( "flat modules", async function( assert ) {
Expand All @@ -334,8 +312,7 @@ QUnit.module( "CLI Main", function() {

assert.equal( execution.code, 0 );
assert.equal( execution.stderr, "" );
const re = new RegExp( expectedOutput[ command ] );
assert.matches( execution.stdout, re );
assert.equal( execution.stdout, expectedOutput[ command ] );
} );
} );
} );

0 comments on commit e295106

Please sign in to comment.