add experimental ESM tracking based on es-module-lexer

assume that other files have no dependencies

lib/FileSystemInfo.js

@@ -16,6 +16,8 @@ const makeSerializable = require("./util/makeSerializable");
 /** @typedef {import("./logging/Logger").Logger} Logger */
 /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
 
+const supportsEsm = +process.versions.modules >= 83;
+
 const resolveContext = createResolver({
 	resolveToContext: true,
 	exportsFields: []
@@ -880,6 +882,8 @@ class FileSystemInfo {
 		this._cachedDeprecatedFileTimestamps = undefined;
 		this._cachedDeprecatedContextTimestamps = undefined;
 
+		this._warnAboutExperimentalEsmTracking = false;
+
 		this._statCreatedSnapshots = 0;
 		this._statTestedSnapshotsCached = 0;
 		this._statTestedSnapshotsNotCached = 0;
@@ -1211,7 +1215,12 @@ class FileSystemInfo {
 						break;
 					}
 					case RBDT_FILE_DEPENDENCIES: {
-						// TODO this probably doesn't work correctly with ESM dependencies
+						// Check for known files without dependencies
+						if (/\.json5?$|\.yarn-integrity$|yarn\.lock$|\.ya?ml/.test(path)) {
+							process.nextTick(callback);
+							break;
+						}
+						// Check commonjs cache for the module
 						/** @type {NodeModule} */
 						const module = require.cache[path];
 						if (module && Array.isArray(module.children)) {
@@ -1248,15 +1257,73 @@ class FileSystemInfo {
 									});
 								}
 							}
+						} else if (supportsEsm && /\.m?js$/.test(path)) {
+							if (!this._warnAboutExperimentalEsmTracking) {
+								this.logger.info(
+									"Node.js doesn't offer a (nice) way to introspect the ESM dependency graph yet.\n" +
+										"Until a full solution is available webpack uses an experimental ESM tracking based on parsing.\n" +
+										"As best effort webpack parses the ESM files to guess dependencies. But this can lead to expensive and incorrect tracking."
+								);
+								this._warnAboutExperimentalEsmTracking = true;
+							}
+							const lexer = require("es-module-lexer");
+							lexer.init.then(() => {
+								this.fs.readFile(path, (err, content) => {
+									if (err) return callback(err);
+									try {
+										const context = dirname(this.fs, path);
+										const source = content.toString();
+										const [imports] = lexer.parse(source);
+										for (const imp of imports) {
+											try {
+												let dependency;
+												if (imp.d === -1) {
+													// import ... from "..."
+													dependency = JSON.parse(
+														source.substring(imp.s - 1, imp.e + 1)
+													);
+												} else if (imp.d > -1) {
+													// import()
+													let expr = source.substring(imp.s, imp.e).trim();
+													if (expr[0] === "'")
+														expr = `"${expr
+															.slice(1, -1)
+															.replace(/"/g, '\\"')}"`;
+													dependency = JSON.parse(expr);
+												} else {
+													// e.g. import.meta
+													continue;
+												}
+												queue.push({
+													type: RBDT_RESOLVE_FILE,
+													context,
+													path: dependency
+												});
+											} catch (e) {
+												this.logger.warn(
+													`Parsing of ${path} for build dependencies failed at 'import(${source.substring(
+														imp.s,
+														imp.e
+													)})'.\n` +
+														"Build dependencies behind this expression are ignored and might cause incorrect cache invalidation."
+												);
+												this.logger.debug(e.stack);
+											}
+										}
+									} catch (e) {
+										this.logger.warn(
+											`Parsing of ${path} for build dependencies failed and all dependencies of this file are ignored, which might cause incorrect cache invalidation..`
+										);
+										this.logger.debug(e.stack);
+									}
+									process.nextTick(callback);
+								});
+							}, callback);
+							break;
 						} else {
-							// Unable to get dependencies from module system
-							// This may be because of an incomplete require.cache implementation like in jest
-							// Assume requires stay in directory and add the whole directory
-							const directory = dirname(this.fs, path);
-							queue.push({
-								type: RBDT_DIRECTORY,
-								path: directory
-							});
+							this.logger.log(
+								`Assuming ${path} has no dependencies as we were unable to assign it to any module system.`
+							);
 						}
 						process.nextTick(callback);
 						break;

package.json

@@ -14,6 +14,7 @@
     "browserslist": "^4.14.5",
     "chrome-trace-event": "^1.0.2",
     "enhanced-resolve": "^5.6.0",
+    "es-module-lexer": "^0.3.26",
     "eslint-scope": "^5.1.1",
     "events": "^3.2.0",
     "glob-to-regexp": "^0.4.1",
@@ -37,6 +38,7 @@
   "devDependencies": {
     "@babel/core": "^7.11.1",
     "@babel/preset-react": "^7.10.4",
+    "@types/es-module-lexer": "^0.3.0",
     "@types/jest": "^26.0.15",
     "@types/node": "^14.14.10",
     "babel-loader": "^8.1.0",

yarn.lock

@@ -836,6 +836,11 @@
   resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd"
   integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==
 
+"@types/es-module-lexer@^0.3.0":
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/@types/es-module-lexer/-/es-module-lexer-0.3.0.tgz#9fee3f19f64e6b3f999eeb3a70bd177a4d57a6cb"
+  integrity sha512-XI3MGSejUQIJ3wzY0i5IHy5J3eb36M/ytgG8jIOssP08ovtRPcjpjXQqrx51AHBNBOisTS/NQNWJitI17+EwzQ==
+
 "@types/eslint-scope@^3.7.0":
   version "3.7.0"
   resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86"
@@ -2345,6 +2350,11 @@ error-ex@^1.3.1:
   dependencies:
     is-arrayish "^0.2.1"
 
+es-module-lexer@^0.3.26:
+  version "0.3.26"
+  resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.3.26.tgz#7b507044e97d5b03b01d4392c74ffeb9c177a83b"
+  integrity sha512-Va0Q/xqtrss45hWzP8CZJwzGSZJjDM5/MJRE3IXXnUCcVLElR9BRaE9F62BopysASyc4nM3uwhSW7FFB9nlWAA==
+
 es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@^0.10.53, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46:
   version "0.10.53"
   resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1"