diff --git a/.gitignore b/.gitignore
index 88b9f186262..28a69f954a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,8 +39,6 @@ lerna-debug.log
 # package-lock file
 package-lock.json
 
-# source maps of docs
-docs/**/*.map
 junit.xml
 
 # typescript source maps
@@ -50,8 +48,3 @@ packages/**/*.map
 # cache
 .eslintcache
 
-# temporary test files
-test-assets/
-./lib/
-
-lib/test/loader/error-test/src/index.d.ts
\ No newline at end of file
diff --git a/packages/webpack-cli/lib/utils/Compiler.js b/packages/webpack-cli/lib/utils/Compiler.js
index 0479354a624..91edf001640 100644
--- a/packages/webpack-cli/lib/utils/Compiler.js
+++ b/packages/webpack-cli/lib/utils/Compiler.js
@@ -87,8 +87,15 @@ class Compiler {
         // eslint-disable-next-line  no-async-promise-executor
         return new Promise(async (resolve) => {
             await this.compiler.run((err, stats) => {
-                const content = this.compilerCallback(err, stats, lastHash, options, outputOptions);
-                resolve(content);
+                if (this.compiler.close) {
+                    this.compiler.close(() => {
+                        const content = this.compilerCallback(err, stats, lastHash, options, outputOptions);
+                        resolve(content);
+                    });
+                } else {
+                    const content = this.compilerCallback(err, stats, lastHash, options, outputOptions);
+                    resolve(content);
+                }
             });
         });
     }
diff --git a/scripts/cleanupTest.js b/scripts/cleanupTest.js
index 43ab80577e0..0f1638df0cf 100644
--- a/scripts/cleanupTest.js
+++ b/scripts/cleanupTest.js
@@ -3,16 +3,27 @@ const rimraf = require('rimraf');
 const { join } = require('path');
 const collectTestFolders = require('./utils');
 
-const outputDirectories = ['bin', 'binary', 'dist', 'test', 'test-assets', 'test-plugin', 'test-loader', 'stats.json'];
+const outputDirectories = [
+    'bin',
+    'binary',
+    'dist',
+    'test',
+    'test-assets',
+    'test-plugin',
+    'test-loader',
+    'test-cache-path',
+    'test-locate-cache',
+    'stats.json',
+];
 
-function folderStrategy(stats, file) {
+const folderStrategy = (stats, file) => {
     return stats.isDirectory() && outputDirectories.includes(file);
-}
+};
 
-function cleanupOutputDirs() {
+const cleanupOutputDirs = () => {
     for (const outputFolder of collectTestFolders(folderStrategy)) {
         outputDirectories.forEach((dir) => rimraf.sync(join(outputFolder, dir)));
     }
-}
+};
 
 module.exports = cleanupOutputDirs;
diff --git a/test/cache/cache.test.js b/test/cache/cache.test.js
new file mode 100644
index 00000000000..8eebde0b56a
--- /dev/null
+++ b/test/cache/cache.test.js
@@ -0,0 +1,16 @@
+'use strict';
+
+const { run, isWebpack5 } = require('../utils/test-utils');
+
+describe('cache related tests', () => {
+    it('should log warning in case of single compiler', () => {
+        let { stderr, stdout } = run(__dirname, ['-c', 'webpack.config.js'], false);
+        // run 2nd compilation
+        ({ stderr, stdout } = run(__dirname, ['-c', 'webpack.config.js'], false));
+
+        if (isWebpack5) {
+            expect(stderr).toContain('starting to restore cache content');
+            expect(stdout).toContain('[cached] 1 module');
+        }
+    });
+});
diff --git a/test/cache/src/main.js b/test/cache/src/main.js
new file mode 100644
index 00000000000..fec83457547
--- /dev/null
+++ b/test/cache/src/main.js
@@ -0,0 +1 @@
+console.log('tehghgst cache');
diff --git a/test/cache/webpack.config.js b/test/cache/webpack.config.js
new file mode 100644
index 00000000000..96c9da4834b
--- /dev/null
+++ b/test/cache/webpack.config.js
@@ -0,0 +1,24 @@
+const path = require('path');
+
+module.exports = {
+    cache: {
+        type: 'filesystem',
+        buildDependencies: {
+            config: [__filename],
+        },
+    },
+    infrastructureLogging: {
+        debug: /webpack\.cache/,
+    },
+    entry: {
+        app: './src/main.js',
+    },
+    devtool: 'inline-source-map',
+    plugins: [],
+    output: {
+        filename: '[name].bundle.js',
+        chunkFilename: '[name].bundle.js',
+        path: path.resolve(__dirname, 'dist'),
+        publicPath: '/',
+    },
+};
diff --git a/test/core-flags/cache-flags.test.js b/test/core-flags/cache-flags.test.js
index c9a3f4bf47f..69cfcd29829 100644
--- a/test/core-flags/cache-flags.test.js
+++ b/test/core-flags/cache-flags.test.js
@@ -1,6 +1,8 @@
 'use strict';
 
 const { run } = require('../utils/test-utils');
+const { existsSync } = require('fs');
+const { resolve } = require('path');
 
 describe('cache related flags from core', () => {
     it('should be successful with --cache ', () => {
@@ -25,17 +27,19 @@ describe('cache related flags from core', () => {
     });
 
     it('should set cache.cacheDirectory with --cache-cache-directory', () => {
-        const { stderr, stdout } = run(__dirname, ['--cache-type', 'filesystem', '--cache-cache-directory', '/test-cache-path']);
+        const { stderr, stdout } = run(__dirname, ['--cache-type', 'filesystem', '--cache-cache-directory', './test-cache-path']);
 
         expect(stderr).toBeFalsy();
         expect(stdout).toContain('test-cache-path');
+        expect(existsSync(resolve(__dirname, './test-cache-path'))).toBeTruthy();
     });
 
     it('should set cache.cacheLocation with --cache-cache-locations', () => {
-        const { stderr, stdout } = run(__dirname, ['--cache-type', 'filesystem', '--cache-cache-location', '/test-locate-cache']);
+        const { stderr, stdout } = run(__dirname, ['--cache-type', 'filesystem', '--cache-cache-location', './test-locate-cache']);
 
         expect(stderr).toBeFalsy();
         expect(stdout).toContain('test-locate-cache');
+        expect(existsSync(resolve(__dirname, './test-locate-cache'))).toBeTruthy();
     });
 
     it('should set cache.hashAlgorithm with --cache-hash-algorithm', () => {
diff --git a/test/loader/error-test/tsconfig.json b/test/loader/error-test/tsconfig.json
new file mode 100644
index 00000000000..27682f8e29b
--- /dev/null
+++ b/test/loader/error-test/tsconfig.json
@@ -0,0 +1,11 @@
+{
+  "compileOnSave": false,
+  "buildOnSave": false,
+  "compilerOptions": {
+    "target": "es6",
+    "moduleResolution": "node",
+    "noEmitOnError": false,
+    "outDir": "./dist"
+  },
+  "exclude": ["node_modules"]
+}
diff --git a/yarn.lock b/yarn.lock
index c9427704a5c..a472e94637f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4067,6 +4067,11 @@ cli-width@^2.0.0:
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639"
   integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=
 
+cli-width@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6"
+  integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==
+
 cliui@^4.0.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49"
@@ -7140,20 +7145,20 @@ inquirer@^6.0.0, inquirer@^6.2.0, inquirer@^6.3.1:
     through "^2.3.6"
 
 inquirer@^7.0.0, inquirer@^7.1.0:
-  version "7.1.0"
-  resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.1.0.tgz#1298a01859883e17c7264b82870ae1034f92dd29"
-  integrity sha512-5fJMWEmikSYu0nv/flMc475MhGbB7TSPd/2IpFV4I4rMklboCH2rQjYY5kKiYGHqUF9gvaambupcJFFG9dvReg==
+  version "7.3.3"
+  resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.3.3.tgz#04d176b2af04afc157a83fd7c100e98ee0aad003"
+  integrity sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==
   dependencies:
     ansi-escapes "^4.2.1"
-    chalk "^3.0.0"
+    chalk "^4.1.0"
     cli-cursor "^3.1.0"
-    cli-width "^2.0.0"
+    cli-width "^3.0.0"
     external-editor "^3.0.3"
     figures "^3.0.0"
-    lodash "^4.17.15"
+    lodash "^4.17.19"
     mute-stream "0.0.8"
     run-async "^2.4.0"
-    rxjs "^6.5.3"
+    rxjs "^6.6.0"
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
     through "^2.3.6"
@@ -11020,17 +11025,10 @@ run-queue@^1.0.0, run-queue@^1.0.3:
   dependencies:
     aproba "^1.1.1"
 
-rxjs@>=6.4.0, rxjs@^6.3.3, rxjs@^6.4.0, rxjs@^6.5.1, rxjs@^6.5.3:
-  version "6.5.4"
-  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.4.tgz#e0777fe0d184cec7872df147f303572d414e211c"
-  integrity sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==
-  dependencies:
-    tslib "^1.9.0"
-
-rxjs@^6.6.2:
-  version "6.6.2"
-  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.2.tgz#8096a7ac03f2cc4fe5860ef6e572810d9e01c0d2"
-  integrity sha512-BHdBMVoWC2sL26w//BCu3YzKT4s2jip/WhwsGEDmeKYBhKDZeYezVUnHatYB7L85v5xs0BAQmg6BEYJEKxBabg==
+rxjs@>=6.4.0, rxjs@^6.3.3, rxjs@^6.4.0, rxjs@^6.5.1, rxjs@^6.6.0, rxjs@^6.6.2:
+  version "6.6.3"
+  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.3.tgz#8ca84635c4daa900c0d3967a6ee7ac60271ee552"
+  integrity sha512-trsQc+xYYXZ3urjOiJOuCOa5N3jAZ3eiSpQB5hIT8zGlL2QfnHLJ2r7GMkBGuIausdJN1OneaI6gQlsqNHHmZQ==
   dependencies:
     tslib "^1.9.0"