Skip to content

Commit

Permalink
More watch test stabilization (#4339)
Browse files Browse the repository at this point in the history 
* Make CLI tests repeatable

* Add watcher to test

* Make it more lightweight by using fs.watch

* More logging

* Use timeouts again

* Remove repetitions
  • Loading branch information
@lukastaegert
lukastaegert committed Jan 5, 2022
1 parent 205e861 commit da3bb43
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 123 deletions.
211 changes: 110 additions & 101 deletions test/cli/index.js
View file
Expand Up @@ -5,7 +5,8 @@ const sander = require('sander');
const {
normaliseOutput,
runTestSuiteWithSamples,
assertDirectoriesAreEqual
assertDirectoriesAreEqual,
getFileNamesAndRemoveOutput
} = require('../utils.js');

const cwd = process.cwd();
Expand All @@ -17,123 +18,131 @@ runTestSuiteWithSamples(
'cli',
path.resolve(__dirname, 'samples'),
(dir, config) => {
(config.skip ? it.skip : config.solo ? it.only : it)(
path.basename(dir) + ': ' + config.description,
done => {
process.chdir(config.cwd || dir);
if (config.before) config.before();
// allow to repeat flaky tests for debugging on CLI
for (let pass = 0; pass < (config.repeat || 1); pass++) {
runTest(dir, config, pass);
}
},
() => process.chdir(cwd)
);

const command = config.command.replace(
/(^| )rollup($| )/g,
`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
);
function runTest(dir, config, pass) {
const name = path.basename(dir) + ': ' + config.description;
(config.skip ? it.skip : config.solo ? it.only : it)(
pass > 0 ? `${name} (pass ${pass + 1})` : name,
done => {
process.chdir(config.cwd || dir);
if (pass > 0) {
getFileNamesAndRemoveOutput(dir);
}
if (config.before) config.before();

const childProcess = exec(
command,
{
timeout: 40000,
env: { ...process.env, FORCE_COLOR: '0', ...config.env }
},
(err, code, stderr) => {
if (config.after) config.after(err, code, stderr);
if (err && !err.killed) {
if (config.error) {
const shouldContinue = config.error(err);
if (!shouldContinue) return done();
} else {
throw err;
}
}
const command = config.command.replace(
/(^| )rollup($| )/g,
`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
);

if ('stderr' in config) {
const shouldContinue = config.stderr(stderr);
const childProcess = exec(
command,
{
timeout: 40000,
env: { ...process.env, FORCE_COLOR: '0', ...config.env }
},
(err, code, stderr) => {
if (config.after) config.after(err, code, stderr);
if (err && !err.killed) {
if (config.error) {
const shouldContinue = config.error(err);
if (!shouldContinue) return done();
} else if (stderr) {
console.error(stderr);
} else {
throw err;
}
}

let unintendedError;

if (config.execute) {
try {
const fn = new Function('require', 'module', 'exports', 'assert', code);
const module = {
exports: {}
};
fn(require, module, module.exports, assert);
if ('stderr' in config) {
const shouldContinue = config.stderr(stderr);
if (!shouldContinue) return done();
} else if (stderr) {
console.error(stderr);
}

if (config.error) {
unintendedError = new Error('Expected an error while executing output');
}
let unintendedError;

if (config.exports) {
config.exports(module.exports);
}
} catch (err) {
if (config.error) {
config.error(err);
} else {
unintendedError = err;
}
}
if (config.execute) {
try {
const fn = new Function('require', 'module', 'exports', 'assert', code);
const module = {
exports: {}
};
fn(require, module, module.exports, assert);

if (config.show || unintendedError) {
console.log(code + '\n\n\n');
if (config.error) {
unintendedError = new Error('Expected an error while executing output');
}

if (config.solo) console.groupEnd();

unintendedError ? done(unintendedError) : done();
} else if (config.result) {
try {
config.result(code);
done();
} catch (err) {
done(err);
}
} else if (config.test) {
try {
config.test();
done();
} catch (err) {
done(err);
}
} else if (
sander.existsSync('_expected') &&
sander.statSync('_expected').isDirectory()
) {
try {
assertDirectoriesAreEqual('_actual', '_expected');
done();
} catch (err) {
done(err);
if (config.exports) {
config.exports(module.exports);
}
} else {
const expected = sander.readFileSync('_expected.js').toString();
try {
assert.equal(normaliseOutput(code), normaliseOutput(expected));
done();
} catch (err) {
done(err);
} catch (err) {
if (config.error) {
config.error(err);
} else {
unintendedError = err;
}
}
}
);

childProcess.stderr.on('data', async data => {
if (config.abortOnStderr) {
if (config.show || unintendedError) {
console.log(code + '\n\n\n');
}

if (config.solo) console.groupEnd();

unintendedError ? done(unintendedError) : done();
} else if (config.result) {
try {
if (await config.abortOnStderr(data)) {
childProcess.kill('SIGTERM');
}
config.result(code);
done();
} catch (err) {
done(err);
}
} else if (config.test) {
try {
config.test();
done();
} catch (err) {
done(err);
}
} else if (sander.existsSync('_expected') && sander.statSync('_expected').isDirectory()) {
try {
assertDirectoriesAreEqual('_actual', '_expected');
done();
} catch (err) {
done(err);
}
} else {
const expected = sander.readFileSync('_expected.js').toString();
try {
assert.equal(normaliseOutput(code), normaliseOutput(expected));
done();
} catch (err) {
childProcess.kill('SIGTERM');
done(err);
}
}
});
}
).timeout(50000);
},
() => process.chdir(cwd)
);
}
);

childProcess.stderr.on('data', async data => {
if (config.abortOnStderr) {
try {
if (await config.abortOnStderr(data)) {
childProcess.kill('SIGTERM');
}
} catch (err) {
childProcess.kill('SIGTERM');
done(err);
}
}
});
}
).timeout(50000);
}
21 changes: 11 additions & 10 deletions test/cli/samples/watch/watch-config-early-update/_config.js
View file
Expand Up @@ -8,7 +8,9 @@ module.exports = {
description: 'immediately reloads the config file if a change happens while it is parsed',
command: 'rollup -cw',
before() {
fs.mkdirSync(path.resolve(__dirname, '_actual'));
// This test writes a config file that prints a message to stderr but delays resolving to a
// config. The stderr message is observed by the parent process and triggers overwriting the
// config file. That way, we simulate a complicated config file being changed while it is parsed.
configFile = path.resolve(__dirname, 'rollup.config.js');
fs.writeFileSync(
configFile,
Expand All @@ -18,16 +20,15 @@ module.exports = {
setTimeout(
() =>
resolve({
input: { output1: 'main.js' },
input: 'main.js',
output: {
dir: '_actual',
file: '_actual/output1.js',
format: 'es'
}
}),
1000
3000
);
});
`
});`
);
},
after() {
Expand All @@ -40,18 +41,18 @@ module.exports = {
`
console.error('updated');
export default {
input: {output2: "main.js"},
input: 'main.js',
output: {
dir: "_actual",
file: '_actual/output2.js',
format: "es"
}
};
`
);
return false;
}
if (data === 'updated\n') {
return new Promise(resolve => setTimeout(() => resolve(true), 500));
if (data.includes(`created _actual${path.sep}output2.js`)) {
return new Promise(resolve => setTimeout(() => resolve(true), 600));
}
}
};
27 changes: 15 additions & 12 deletions test/cli/samples/watch/watch-config-error/_config.js
View file
Expand Up @@ -11,22 +11,25 @@ module.exports = {
configFile = path.resolve(__dirname, 'rollup.config.js');
fs.writeFileSync(
configFile,
'export default {\n' +
'\tinput: "main.js",\n' +
'\toutput: {\n' +
'\t\tfile: "_actual/main1.js",\n' +
'\t\tformat: "es"\n' +
'\t}\n' +
'};'
`
export default {
input: "main.js",
output: {
file: "_actual/main1.js",
format: "es"
}
};`
);
},
after() {
// synchronous sometimes does not seem to work, probably because the watch is not yet removed properly
setTimeout(() => fs.unlinkSync(configFile), 300);
fs.unlinkSync(configFile);
},
abortOnStderr(data) {
if (data.includes(`created _actual${path.sep}main1.js`)) {
atomicWriteFileSync(configFile, 'throw new Error("Config contains errors");');
setTimeout(
() => atomicWriteFileSync(configFile, 'throw new Error("Config contains errors");'),
600
);
return false;
}
if (data.includes('Config contains errors')) {
Expand All @@ -41,11 +44,11 @@ module.exports = {
'\t}\n' +
'};'
);
}, 400);
}, 600);
return false;
}
if (data.includes(`created _actual${path.sep}main2.js`)) {
return true;
return new Promise(resolve => setTimeout(() => resolve(true), 600));
}
}
};
1 change: 1 addition & 0 deletions test/utils.js
View file
Expand Up @@ -16,6 +16,7 @@ exports.assertDirectoriesAreEqual = assertDirectoriesAreEqual;
exports.assertFilesAreEqual = assertFilesAreEqual;
exports.assertIncludes = assertIncludes;
exports.atomicWriteFileSync = atomicWriteFileSync;
exports.getFileNamesAndRemoveOutput = getFileNamesAndRemoveOutput;

function normaliseError(error) {
delete error.stack;
Expand Down

0 comments on commit da3bb43

Please sign in to comment.