diff --git a/test/ConfigTestCases.template.js b/test/ConfigTestCases.template.js
index f3d3f81fc0f..2ab0636e49e 100644
--- a/test/ConfigTestCases.template.js
+++ b/test/ConfigTestCases.template.js
@@ -572,16 +572,20 @@ const describeCases = config => {
 									.catch(done);
 							};
 							if (config.cache) {
-								const compiler = require("..")(options);
-								compiler.run(err => {
-									if (err) return handleFatalError(err, done);
-									compiler.run((error, stats) => {
-										compiler.close(err => {
-											if (err) return handleFatalError(err, done);
-											onCompiled(error, stats);
+								try {
+									const compiler = require("..")(options);
+									compiler.run(err => {
+										if (err) return handleFatalError(err, done);
+										compiler.run((error, stats) => {
+											compiler.close(err => {
+												if (err) return handleFatalError(err, done);
+												onCompiled(error, stats);
+											});
 										});
 									});
-								});
+								} catch (e) {
+									handleFatalError(e, done);
+								}
 							} else {
 								require("..")(options, onCompiled);
 							}
diff --git a/test/configCases/asset-modules/http-url/server/index.js b/test/configCases/asset-modules/http-url/server/index.js
index e48a3072fce..1ab7e59e66e 100644
--- a/test/configCases/asset-modules/http-url/server/index.js
+++ b/test/configCases/asset-modules/http-url/server/index.js
@@ -6,7 +6,10 @@ const fs = require("fs");
  * @returns {Promise<import("http").Server>} server instance
  */
 function createServer(port) {
-	const file = fs.readFileSync("./test/configCases/asset-modules/http-url/server/index.css").toString().trim();
+	const file = fs
+		.readFileSync("./test/configCases/asset-modules/http-url/server/index.css")
+		.toString()
+		.trim();
 
 	const server = http.createServer((req, res) => {
 		if (req.url !== "/index.css") {
@@ -18,7 +21,7 @@ function createServer(port) {
 	});
 
 	return new Promise((resolve, reject) => {
-		server.listen(port, (err) => {
+		server.listen(port, err => {
 			if (err) {
 				reject(err);
 			} else {
@@ -40,21 +43,25 @@ class ServerPlugin {
 	 * @param {import("../../../../../").Compiler} compiler
 	 */
 	apply(compiler) {
-		const serverPromise = createServer(this.port);
+		let server;
 
-		serverPromise
-			.then(server => server.unref());
+		compiler.hooks.beforeRun.tapPromise(
+			"ServerPlugin",
+			async (compiler, callback) => {
+				if (!server) {
+					server = await createServer(this.port);
+					server.unref();
+				}
+			}
+		);
 
 		compiler.hooks.done.tapAsync("ServerPlugin", (stats, callback) => {
-			serverPromise
-				.then(server => server.close(callback))
-				.catch(callback)
-		});
-
-		compiler.hooks.beforeRun.tapAsync("ServerPlugin", (compiler, callback) => {
-			serverPromise
-				.then(() => callback())
-				.catch(callback)
+			if (server) {
+				server.close(callback);
+				server = undefined;
+			} else {
+				callback();
+			}
 		});
 	}
 }
diff --git a/test/configCases/clean/enabled/webpack.config.js b/test/configCases/clean/enabled/webpack.config.js
index ade18386541..16575014af3 100644
--- a/test/configCases/clean/enabled/webpack.config.js
+++ b/test/configCases/clean/enabled/webpack.config.js
@@ -10,12 +10,19 @@ module.exports = {
 	},
 	plugins: [
 		compiler => {
+			let once = true;
 			compiler.hooks.thisCompilation.tap("Test", compilation => {
 				compilation.hooks.processAssets.tap("Test", assets => {
-					const outputPath = compilation.getPath(compiler.outputPath, {});
-					const customDir = path.join(outputPath, "this/dir/should/be/removed");
-					fs.mkdirSync(customDir, { recursive: true });
-					fs.writeFileSync(path.join(customDir, "file.ext"), "");
+					if (once) {
+						const outputPath = compilation.getPath(compiler.outputPath, {});
+						const customDir = path.join(
+							outputPath,
+							"this/dir/should/be/removed"
+						);
+						fs.mkdirSync(customDir, { recursive: true });
+						fs.writeFileSync(path.join(customDir, "file.ext"), "");
+						once = false;
+					}
 					assets["this/dir/should/not/be/removed/file.ext"] = new RawSource("");
 				});
 			});
diff --git a/test/configCases/clean/ignore-fn/webpack.config.js b/test/configCases/clean/ignore-fn/webpack.config.js
index e9a42c5c2b5..387174ab8e4 100644
--- a/test/configCases/clean/ignore-fn/webpack.config.js
+++ b/test/configCases/clean/ignore-fn/webpack.config.js
@@ -14,18 +14,25 @@ module.exports = {
 	},
 	plugins: [
 		compiler => {
+			let once = true;
 			compiler.hooks.thisCompilation.tap("Test", compilation => {
 				compilation.hooks.processAssets.tap("Test", assets => {
-					const outputPath = compilation.getPath(compiler.outputPath, {});
-					const customDir = path.join(outputPath, "this/dir/should/be/removed");
-					const ignoredDir = path.join(
-						outputPath,
-						"this/is/ignored/dir/that/should/not/be/removed"
-					);
-					fs.mkdirSync(customDir, { recursive: true });
-					fs.writeFileSync(path.join(customDir, "file.ext"), "");
-					fs.mkdirSync(ignoredDir, { recursive: true });
-					fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
+					if (once) {
+						const outputPath = compilation.getPath(compiler.outputPath, {});
+						const customDir = path.join(
+							outputPath,
+							"this/dir/should/be/removed"
+						);
+						const ignoredDir = path.join(
+							outputPath,
+							"this/is/ignored/dir/that/should/not/be/removed"
+						);
+						fs.mkdirSync(customDir, { recursive: true });
+						fs.writeFileSync(path.join(customDir, "file.ext"), "");
+						fs.mkdirSync(ignoredDir, { recursive: true });
+						fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
+						once = false;
+					}
 					assets["this/dir/should/not/be/removed/file.ext"] = new RawSource("");
 				});
 			});
diff --git a/test/configCases/clean/ignore-hook/webpack.config.js b/test/configCases/clean/ignore-hook/webpack.config.js
index 7cdf7734043..caee5cf6a09 100644
--- a/test/configCases/clean/ignore-hook/webpack.config.js
+++ b/test/configCases/clean/ignore-hook/webpack.config.js
@@ -11,6 +11,7 @@ module.exports = {
 	},
 	plugins: [
 		compiler => {
+			let once = true;
 			compiler.hooks.thisCompilation.tap("Test", compilation => {
 				webpack.CleanPlugin.getCompilationHooks(compilation).keep.tap(
 					"Test",
@@ -20,22 +21,28 @@ module.exports = {
 					}
 				);
 				compilation.hooks.processAssets.tap("Test", assets => {
-					const outputPath = compilation.getPath(compiler.outputPath, {});
-					const customDir = path.join(outputPath, "this/dir/should/be/removed");
-					const ignoredDir = path.join(
-						outputPath,
-						"this/is/ignored/dir/that/should/not/be/removed"
-					);
-					const ignoredTooDir = path.join(
-						outputPath,
-						"this/is/ignored/too/dir/that/should/not/be/removed"
-					);
-					fs.mkdirSync(customDir, { recursive: true });
-					fs.writeFileSync(path.join(customDir, "file.ext"), "");
-					fs.mkdirSync(ignoredDir, { recursive: true });
-					fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
-					fs.mkdirSync(ignoredTooDir, { recursive: true });
-					fs.writeFileSync(path.join(ignoredTooDir, "file.ext"), "");
+					if (once) {
+						const outputPath = compilation.getPath(compiler.outputPath, {});
+						const customDir = path.join(
+							outputPath,
+							"this/dir/should/be/removed"
+						);
+						const ignoredDir = path.join(
+							outputPath,
+							"this/is/ignored/dir/that/should/not/be/removed"
+						);
+						const ignoredTooDir = path.join(
+							outputPath,
+							"this/is/ignored/too/dir/that/should/not/be/removed"
+						);
+						fs.mkdirSync(customDir, { recursive: true });
+						fs.writeFileSync(path.join(customDir, "file.ext"), "");
+						fs.mkdirSync(ignoredDir, { recursive: true });
+						fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
+						fs.mkdirSync(ignoredTooDir, { recursive: true });
+						fs.writeFileSync(path.join(ignoredTooDir, "file.ext"), "");
+						once = false;
+					}
 					assets["this/dir/should/not/be/removed/file.ext"] = new RawSource("");
 				});
 			});
diff --git a/test/configCases/clean/ignore-rx/webpack.config.js b/test/configCases/clean/ignore-rx/webpack.config.js
index 9a4d6abe6f1..82623ce620c 100644
--- a/test/configCases/clean/ignore-rx/webpack.config.js
+++ b/test/configCases/clean/ignore-rx/webpack.config.js
@@ -12,18 +12,25 @@ module.exports = {
 	},
 	plugins: [
 		compiler => {
+			let once = true;
 			compiler.hooks.thisCompilation.tap("Test", compilation => {
 				compilation.hooks.processAssets.tap("Test", assets => {
-					const outputPath = compilation.getPath(compiler.outputPath, {});
-					const customDir = path.join(outputPath, "this/dir/should/be/removed");
-					const ignoredDir = path.join(
-						outputPath,
-						"this/is/ignored/dir/that/should/not/be/removed"
-					);
-					fs.mkdirSync(customDir, { recursive: true });
-					fs.writeFileSync(path.join(customDir, "file.ext"), "");
-					fs.mkdirSync(ignoredDir, { recursive: true });
-					fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
+					if (once) {
+						const outputPath = compilation.getPath(compiler.outputPath, {});
+						const customDir = path.join(
+							outputPath,
+							"this/dir/should/be/removed"
+						);
+						const ignoredDir = path.join(
+							outputPath,
+							"this/is/ignored/dir/that/should/not/be/removed"
+						);
+						fs.mkdirSync(customDir, { recursive: true });
+						fs.writeFileSync(path.join(customDir, "file.ext"), "");
+						fs.mkdirSync(ignoredDir, { recursive: true });
+						fs.writeFileSync(path.join(ignoredDir, "file.ext"), "");
+						once = false;
+					}
 					assets["this/dir/should/not/be/removed/file.ext"] = new RawSource("");
 				});
 			});