diff --git a/test/cli/index.js b/test/cli/index.js
index 4bbd525a431..7f747a34024 100644
--- a/test/cli/index.js
+++ b/test/cli/index.js
@@ -5,7 +5,8 @@ const sander = require('sander');
 const {
 	normaliseOutput,
 	runTestSuiteWithSamples,
-	assertDirectoriesAreEqual
+	assertDirectoriesAreEqual,
+	getFileNamesAndRemoveOutput
 } = require('../utils.js');
 
 const cwd = process.cwd();
@@ -17,123 +18,131 @@ runTestSuiteWithSamples(
 	'cli',
 	path.resolve(__dirname, 'samples'),
 	(dir, config) => {
-		(config.skip ? it.skip : config.solo ? it.only : it)(
-			path.basename(dir) + ': ' + config.description,
-			done => {
-				process.chdir(config.cwd || dir);
-				if (config.before) config.before();
+		// allow to repeat flaky tests for debugging on CLI
+		for (let pass = 0; pass < (config.repeat || 1); pass++) {
+			runTest(dir, config, pass);
+		}
+	},
+	() => process.chdir(cwd)
+);
 
-				const command = config.command.replace(
-					/(^| )rollup($| )/g,
-					`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
-				);
+function runTest(dir, config, pass) {
+	const name = path.basename(dir) + ': ' + config.description;
+	(config.skip ? it.skip : config.solo ? it.only : it)(
+		pass > 0 ? `${name} (pass ${pass + 1})` : name,
+		done => {
+			process.chdir(config.cwd || dir);
+			if (pass > 0) {
+				getFileNamesAndRemoveOutput(dir);
+			}
+			if (config.before) config.before();
 
-				const childProcess = exec(
-					command,
-					{
-						timeout: 40000,
-						env: { ...process.env, FORCE_COLOR: '0', ...config.env }
-					},
-					(err, code, stderr) => {
-						if (config.after) config.after(err, code, stderr);
-						if (err && !err.killed) {
-							if (config.error) {
-								const shouldContinue = config.error(err);
-								if (!shouldContinue) return done();
-							} else {
-								throw err;
-							}
-						}
+			const command = config.command.replace(
+				/(^| )rollup($| )/g,
+				`node ${path.resolve(__dirname, '../../dist/bin')}${path.sep}rollup `
+			);
 
-						if ('stderr' in config) {
-							const shouldContinue = config.stderr(stderr);
+			const childProcess = exec(
+				command,
+				{
+					timeout: 40000,
+					env: { ...process.env, FORCE_COLOR: '0', ...config.env }
+				},
+				(err, code, stderr) => {
+					if (config.after) config.after(err, code, stderr);
+					if (err && !err.killed) {
+						if (config.error) {
+							const shouldContinue = config.error(err);
 							if (!shouldContinue) return done();
-						} else if (stderr) {
-							console.error(stderr);
+						} else {
+							throw err;
 						}
+					}
 
-						let unintendedError;
-
-						if (config.execute) {
-							try {
-								const fn = new Function('require', 'module', 'exports', 'assert', code);
-								const module = {
-									exports: {}
-								};
-								fn(require, module, module.exports, assert);
+					if ('stderr' in config) {
+						const shouldContinue = config.stderr(stderr);
+						if (!shouldContinue) return done();
+					} else if (stderr) {
+						console.error(stderr);
+					}
 
-								if (config.error) {
-									unintendedError = new Error('Expected an error while executing output');
-								}
+					let unintendedError;
 
-								if (config.exports) {
-									config.exports(module.exports);
-								}
-							} catch (err) {
-								if (config.error) {
-									config.error(err);
-								} else {
-									unintendedError = err;
-								}
-							}
+					if (config.execute) {
+						try {
+							const fn = new Function('require', 'module', 'exports', 'assert', code);
+							const module = {
+								exports: {}
+							};
+							fn(require, module, module.exports, assert);
 
-							if (config.show || unintendedError) {
-								console.log(code + '\n\n\n');
+							if (config.error) {
+								unintendedError = new Error('Expected an error while executing output');
 							}
 
-							if (config.solo) console.groupEnd();
-
-							unintendedError ? done(unintendedError) : done();
-						} else if (config.result) {
-							try {
-								config.result(code);
-								done();
-							} catch (err) {
-								done(err);
-							}
-						} else if (config.test) {
-							try {
-								config.test();
-								done();
-							} catch (err) {
-								done(err);
-							}
-						} else if (
-							sander.existsSync('_expected') &&
-							sander.statSync('_expected').isDirectory()
-						) {
-							try {
-								assertDirectoriesAreEqual('_actual', '_expected');
-								done();
-							} catch (err) {
-								done(err);
+							if (config.exports) {
+								config.exports(module.exports);
 							}
-						} else {
-							const expected = sander.readFileSync('_expected.js').toString();
-							try {
-								assert.equal(normaliseOutput(code), normaliseOutput(expected));
-								done();
-							} catch (err) {
-								done(err);
+						} catch (err) {
+							if (config.error) {
+								config.error(err);
+							} else {
+								unintendedError = err;
 							}
 						}
-					}
-				);
 
-				childProcess.stderr.on('data', async data => {
-					if (config.abortOnStderr) {
+						if (config.show || unintendedError) {
+							console.log(code + '\n\n\n');
+						}
+
+						if (config.solo) console.groupEnd();
+
+						unintendedError ? done(unintendedError) : done();
+					} else if (config.result) {
 						try {
-							if (await config.abortOnStderr(data)) {
-								childProcess.kill('SIGTERM');
-							}
+							config.result(code);
+							done();
+						} catch (err) {
+							done(err);
+						}
+					} else if (config.test) {
+						try {
+							config.test();
+							done();
+						} catch (err) {
+							done(err);
+						}
+					} else if (sander.existsSync('_expected') && sander.statSync('_expected').isDirectory()) {
+						try {
+							assertDirectoriesAreEqual('_actual', '_expected');
+							done();
+						} catch (err) {
+							done(err);
+						}
+					} else {
+						const expected = sander.readFileSync('_expected.js').toString();
+						try {
+							assert.equal(normaliseOutput(code), normaliseOutput(expected));
+							done();
 						} catch (err) {
-							childProcess.kill('SIGTERM');
 							done(err);
 						}
 					}
-				});
-			}
-		).timeout(50000);
-	},
-	() => process.chdir(cwd)
-);
+				}
+			);
+
+			childProcess.stderr.on('data', async data => {
+				if (config.abortOnStderr) {
+					try {
+						if (await config.abortOnStderr(data)) {
+							childProcess.kill('SIGTERM');
+						}
+					} catch (err) {
+						childProcess.kill('SIGTERM');
+						done(err);
+					}
+				}
+			});
+		}
+	).timeout(50000);
+}
diff --git a/test/cli/samples/watch/watch-config-early-update/_config.js b/test/cli/samples/watch/watch-config-early-update/_config.js
index d7b769c9058..7451c2f3492 100644
--- a/test/cli/samples/watch/watch-config-early-update/_config.js
+++ b/test/cli/samples/watch/watch-config-early-update/_config.js
@@ -5,6 +5,7 @@ const { atomicWriteFileSync } = require('../../../../utils');
 let configFile;
 
 module.exports = {
+	repeat: 10,
 	description: 'immediately reloads the config file if a change happens while it is parsed',
 	command: 'rollup -cw',
 	before() {
diff --git a/test/utils.js b/test/utils.js
index 23aecff9d3a..d8896c1d306 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -16,6 +16,7 @@ exports.assertDirectoriesAreEqual = assertDirectoriesAreEqual;
 exports.assertFilesAreEqual = assertFilesAreEqual;
 exports.assertIncludes = assertIncludes;
 exports.atomicWriteFileSync = atomicWriteFileSync;
+exports.getFileNamesAndRemoveOutput = getFileNamesAndRemoveOutput;
 
 function normaliseError(error) {
 	delete error.stack;