diff --git a/lib/FileSystemInfo.js b/lib/FileSystemInfo.js index 16d894df29c..590929c9242 100644 --- a/lib/FileSystemInfo.js +++ b/lib/FileSystemInfo.js @@ -16,6 +16,8 @@ const makeSerializable = require("./util/makeSerializable"); /** @typedef {import("./logging/Logger").Logger} Logger */ /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */ +const supportsEsm = +process.versions.modules >= 83; + const resolveContext = createResolver({ resolveToContext: true, exportsFields: [] @@ -880,6 +882,8 @@ class FileSystemInfo { this._cachedDeprecatedFileTimestamps = undefined; this._cachedDeprecatedContextTimestamps = undefined; + this._warnAboutExperimentalEsmTracking = false; + this._statCreatedSnapshots = 0; this._statTestedSnapshotsCached = 0; this._statTestedSnapshotsNotCached = 0; @@ -1211,7 +1215,12 @@ class FileSystemInfo { break; } case RBDT_FILE_DEPENDENCIES: { - // TODO this probably doesn't work correctly with ESM dependencies + // Check for known files without dependencies + if (/\.json5?$|\.yarn-integrity$|yarn\.lock$|\.ya?ml/.test(path)) { + process.nextTick(callback); + break; + } + // Check commonjs cache for the module /** @type {NodeModule} */ const module = require.cache[path]; if (module && Array.isArray(module.children)) { @@ -1248,15 +1257,73 @@ class FileSystemInfo { }); } } + } else if (supportsEsm && /\.m?js$/.test(path)) { + if (!this._warnAboutExperimentalEsmTracking) { + this.logger.info( + "Node.js doesn't offer a (nice) way to introspect the ESM dependency graph yet.\n" + + "Until a full solution is available webpack uses an experimental ESM tracking based on parsing.\n" + + "As best effort webpack parses the ESM files to guess dependencies. But this can lead to expensive and incorrect tracking." + ); + this._warnAboutExperimentalEsmTracking = true; + } + const lexer = require("es-module-lexer"); + lexer.init.then(() => { + this.fs.readFile(path, (err, content) => { + if (err) return callback(err); + try { + const context = dirname(this.fs, path); + const source = content.toString(); + const [imports] = lexer.parse(source); + for (const imp of imports) { + try { + let dependency; + if (imp.d === -1) { + // import ... from "..." + dependency = JSON.parse( + source.substring(imp.s - 1, imp.e + 1) + ); + } else if (imp.d > -1) { + // import() + let expr = source.substring(imp.s, imp.e).trim(); + if (expr[0] === "'") + expr = `"${expr + .slice(1, -1) + .replace(/"/g, '\\"')}"`; + dependency = JSON.parse(expr); + } else { + // e.g. import.meta + continue; + } + queue.push({ + type: RBDT_RESOLVE_FILE, + context, + path: dependency + }); + } catch (e) { + this.logger.warn( + `Parsing of ${path} for build dependencies failed at 'import(${source.substring( + imp.s, + imp.e + )})'.\n` + + "Build dependencies behind this expression are ignored and might cause incorrect cache invalidation." + ); + this.logger.debug(e.stack); + } + } + } catch (e) { + this.logger.warn( + `Parsing of ${path} for build dependencies failed and all dependencies of this file are ignored, which might cause incorrect cache invalidation..` + ); + this.logger.debug(e.stack); + } + process.nextTick(callback); + }); + }, callback); + break; } else { - // Unable to get dependencies from module system - // This may be because of an incomplete require.cache implementation like in jest - // Assume requires stay in directory and add the whole directory - const directory = dirname(this.fs, path); - queue.push({ - type: RBDT_DIRECTORY, - path: directory - }); + this.logger.log( + `Assuming ${path} has no dependencies as we were unable to assign it to any module system.` + ); } process.nextTick(callback); break; diff --git a/package.json b/package.json index 9df900cacee..5789e7d7b8a 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.6.0", + "es-module-lexer": "^0.3.26", "eslint-scope": "^5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", @@ -37,6 +38,7 @@ "devDependencies": { "@babel/core": "^7.11.1", "@babel/preset-react": "^7.10.4", + "@types/es-module-lexer": "^0.3.0", "@types/jest": "^26.0.15", "@types/node": "^14.14.10", "babel-loader": "^8.1.0", diff --git a/yarn.lock b/yarn.lock index 97630c2c6f4..ebc1251cf4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -836,6 +836,11 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== +"@types/es-module-lexer@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@types/es-module-lexer/-/es-module-lexer-0.3.0.tgz#9fee3f19f64e6b3f999eeb3a70bd177a4d57a6cb" + integrity sha512-XI3MGSejUQIJ3wzY0i5IHy5J3eb36M/ytgG8jIOssP08ovtRPcjpjXQqrx51AHBNBOisTS/NQNWJitI17+EwzQ== + "@types/eslint-scope@^3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86" @@ -2345,6 +2350,11 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +es-module-lexer@^0.3.26: + version "0.3.26" + resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.3.26.tgz#7b507044e97d5b03b01d4392c74ffeb9c177a83b" + integrity sha512-Va0Q/xqtrss45hWzP8CZJwzGSZJjDM5/MJRE3IXXnUCcVLElR9BRaE9F62BopysASyc4nM3uwhSW7FFB9nlWAA== + es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@^0.10.53, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46: version "0.10.53" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1"