Skip to content

Commit

Permalink
Make CLI tests repeatable
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukastaegert
lukastaegert committed Jan 4, 2022
1 parent ae674c9 commit 95ee894
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 101 deletions.
211 changes: 110 additions & 101 deletions test/cli/index.js
View file
Expand Up @@ -5,7 +5,8 @@ const sander = require('sander');
const {
normaliseOutput,
runTestSuiteWithSamples,
assertDirectoriesAreEqual
assertDirectoriesAreEqual,
getFileNamesAndRemoveOutput
} = require('../utils.js');

const cwd = process.cwd();
Expand All @@ -17,123 +18,131 @@ runTestSuiteWithSamples(
'cli',
path.resolve(__dirname, 'samples'),
(dir, config) => {
(config.skip ? it.skip : config.solo ? it.only : it)(
path.basename(dir) + ': ' + config.description,
done => {
process.chdir(config.cwd || dir);
if (config.before) config.before();
// allow to repeat flaky tests for debugging on CLI
for (let pass = 0; pass < (config.repeat || 1); pass++) {
runTest(dir, config, pass);
}
},
() => process.chdir(cwd)
);

const command = config.command.replace(
/(^| )rollup($| )/g,
`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
);
function runTest(dir, config, pass) {
const name = path.basename(dir) + ': ' + config.description;
(config.skip ? it.skip : config.solo ? it.only : it)(
pass > 0 ? `${name} (pass ${pass + 1})` : name,
done => {
process.chdir(config.cwd || dir);
if (pass > 0) {
getFileNamesAndRemoveOutput(dir);
}
if (config.before) config.before();

const childProcess = exec(
command,
{
timeout: 40000,
env: { ...process.env, FORCE_COLOR: '0', ...config.env }
},
(err, code, stderr) => {
if (config.after) config.after(err, code, stderr);
if (err && !err.killed) {
if (config.error) {
const shouldContinue = config.error(err);
if (!shouldContinue) return done();
} else {
throw err;
}
}
const command = config.command.replace(
/(^| )rollup($| )/g,
`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
);

if ('stderr' in config) {
const shouldContinue = config.stderr(stderr);
const childProcess = exec(
command,
{
timeout: 40000,
env: { ...process.env, FORCE_COLOR: '0', ...config.env }
},
(err, code, stderr) => {
if (config.after) config.after(err, code, stderr);
if (err && !err.killed) {
if (config.error) {
const shouldContinue = config.error(err);
if (!shouldContinue) return done();
} else if (stderr) {
console.error(stderr);
} else {
throw err;
}
}

let unintendedError;

if (config.execute) {
try {
const fn = new Function('require', 'module', 'exports', 'assert', code);
const module = {
exports: {}
};
fn(require, module, module.exports, assert);
if ('stderr' in config) {
const shouldContinue = config.stderr(stderr);
if (!shouldContinue) return done();
} else if (stderr) {
console.error(stderr);
}

if (config.error) {
unintendedError = new Error('Expected an error while executing output');
}
let unintendedError;

if (config.exports) {
config.exports(module.exports);
}
} catch (err) {
if (config.error) {
config.error(err);
} else {
unintendedError = err;
}
}
if (config.execute) {
try {
const fn = new Function('require', 'module', 'exports', 'assert', code);
const module = {
exports: {}
};
fn(require, module, module.exports, assert);

if (config.show || unintendedError) {
console.log(code + '\n\n\n');
if (config.error) {
unintendedError = new Error('Expected an error while executing output');
}

if (config.solo) console.groupEnd();

unintendedError ? done(unintendedError) : done();
} else if (config.result) {
try {
config.result(code);
done();
} catch (err) {
done(err);
}
} else if (config.test) {
try {
config.test();
done();
} catch (err) {
done(err);
}
} else if (
sander.existsSync('_expected') &&
sander.statSync('_expected').isDirectory()
) {
try {
assertDirectoriesAreEqual('_actual', '_expected');
done();
} catch (err) {
done(err);
if (config.exports) {
config.exports(module.exports);
}
} else {
const expected = sander.readFileSync('_expected.js').toString();
try {
assert.equal(normaliseOutput(code), normaliseOutput(expected));
done();
} catch (err) {
done(err);
} catch (err) {
if (config.error) {
config.error(err);
} else {
unintendedError = err;
}
}
}
);

childProcess.stderr.on('data', async data => {
if (config.abortOnStderr) {
if (config.show || unintendedError) {
console.log(code + '\n\n\n');
}

if (config.solo) console.groupEnd();

unintendedError ? done(unintendedError) : done();
} else if (config.result) {
try {
if (await config.abortOnStderr(data)) {
childProcess.kill('SIGTERM');
}
config.result(code);
done();
} catch (err) {
done(err);
}
} else if (config.test) {
try {
config.test();
done();
} catch (err) {
done(err);
}
} else if (sander.existsSync('_expected') && sander.statSync('_expected').isDirectory()) {
try {
assertDirectoriesAreEqual('_actual', '_expected');
done();
} catch (err) {
done(err);
}
} else {
const expected = sander.readFileSync('_expected.js').toString();
try {
assert.equal(normaliseOutput(code), normaliseOutput(expected));
done();
} catch (err) {
childProcess.kill('SIGTERM');
done(err);
}
}
});
}
).timeout(50000);
},
() => process.chdir(cwd)
);
}
);

childProcess.stderr.on('data', async data => {
if (config.abortOnStderr) {
try {
if (await config.abortOnStderr(data)) {
childProcess.kill('SIGTERM');
}
} catch (err) {
childProcess.kill('SIGTERM');
done(err);
}
}
});
}
).timeout(50000);
}
2 changes: 2 additions & 0 deletions test/cli/samples/watch/watch-config-early-update/_config.js
View file
Expand Up @@ -5,6 +5,8 @@ const { atomicWriteFileSync } = require('../../../../utils');
let configFile;

module.exports = {
solo: true,
repeat: 10,
description: 'immediately reloads the config file if a change happens while it is parsed',
command: 'rollup -cw',
before() {
Expand Down
1 change: 1 addition & 0 deletions test/utils.js
View file
Expand Up @@ -16,6 +16,7 @@ exports.assertDirectoriesAreEqual = assertDirectoriesAreEqual;
exports.assertFilesAreEqual = assertFilesAreEqual;
exports.assertIncludes = assertIncludes;
exports.atomicWriteFileSync = atomicWriteFileSync;
exports.getFileNamesAndRemoveOutput = getFileNamesAndRemoveOutput;

function normaliseError(error) {
delete error.stack;
Expand Down

0 comments on commit 95ee894

Please sign in to comment.