From 50844eae3b77b3b5e127f82a13ca2506f4fd9ab6 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 13 Jan 2021 20:35:33 +0100 Subject: [PATCH 1/3] add test case for ESM tracking --- test/BuildDependencies.test.js | 54 +++++++- .../fixtures/buildDependencies/esm-cjs-dep.js | 1 + test/fixtures/buildDependencies/esm-dep.mjs | 1 + test/fixtures/buildDependencies/esm.mjs | 5 + test/fixtures/buildDependencies/index.js | 2 + test/fixtures/buildDependencies/run.js | 124 ++++++++++-------- 6 files changed, 128 insertions(+), 59 deletions(-) create mode 100644 test/fixtures/buildDependencies/esm-cjs-dep.js create mode 100644 test/fixtures/buildDependencies/esm-dep.mjs create mode 100644 test/fixtures/buildDependencies/esm.mjs diff --git a/test/BuildDependencies.test.js b/test/BuildDependencies.test.js index 2228012e236..fcf227a4233 100644 --- a/test/BuildDependencies.test.js +++ b/test/BuildDependencies.test.js @@ -24,7 +24,6 @@ const exec = (n, options = {}) => { if (code === 0) { if (!options.ignoreErrors && /<[ew]>/.test(stdout)) return reject(stdout); - console.log(stdout); resolve(stdout); } else { reject(new Error(`Code ${code}: ${stdout}`)); @@ -34,6 +33,8 @@ const exec = (n, options = {}) => { }); }; +const supportsEsm = +process.versions.modules >= 83; + describe("BuildDependencies", () => { beforeEach(done => { rimraf(cacheDirectory, done); @@ -57,6 +58,14 @@ describe("BuildDependencies", () => { path.resolve(inputDirectory, "config-dependency.js"), "module.exports = 0;" ); + fs.writeFileSync( + path.resolve(inputDirectory, "esm-dependency.js"), + "module.exports = 0;" + ); + fs.writeFileSync( + path.resolve(inputDirectory, "esm-async-dependency.mjs"), + "export default 0;" + ); await exec("0", { invalidBuildDepdencies: true, buildTwice: true, @@ -70,14 +79,18 @@ describe("BuildDependencies", () => { path.resolve(inputDirectory, "config-dependency.js"), "module.exports = 1;" ); + fs.writeFileSync( + path.resolve(inputDirectory, "esm-dependency.js"), + "module.exports = 1;" + ); await exec("1"); fs.writeFileSync( path.resolve(inputDirectory, "loader-dependency.js"), "module.exports = Date.now();" ); const now1 = Date.now(); - await exec("2"); - await exec("3"); + expect(await exec("2")).toMatch(/Captured build dependencies/); + expect(await exec("3")).not.toMatch(/Captured build dependencies/); fs.writeFileSync( path.resolve(inputDirectory, "config-dependency"), "module.exports = Date.now();" @@ -86,7 +99,22 @@ describe("BuildDependencies", () => { await exec("4"); const now3 = Date.now(); await exec("5"); - const results = Array.from({ length: 6 }).map((_, i) => + let now4, now5; + if (supportsEsm) { + fs.writeFileSync( + path.resolve(inputDirectory, "esm-dependency.js"), + "module.exports = Date.now();" + ); + now4 = Date.now(); + await exec("6"); + fs.writeFileSync( + path.resolve(inputDirectory, "esm-async-dependency.mjs"), + "export default Date.now();" + ); + now5 = Date.now(); + await exec("7"); + } + const results = Array.from({ length: supportsEsm ? 8 : 6 }).map((_, i) => require(`./js/buildDeps/${i}/main.js`) ); for (const r of results) { @@ -96,26 +124,44 @@ describe("BuildDependencies", () => { } expect(results[0].loader).toBe(0); expect(results[0].config).toBe(0); + if (supportsEsm) expect(results[0].esmConfig).toBe(0); expect(results[0].uncached).toBe(0); // 0 -> 1 should not cache at all because of invalid buildDeps expect(results[1].loader).toBe(1); expect(results[1].config).toBe(1); + expect(results[1].esmConfig).toBe(1); expect(results[1].uncached).toBe(1); // 1 -> 2 should be invalidated expect(results[2].loader).toBeGreaterThan(now1); expect(results[2].config).toBe(1); + expect(results[2].esmConfig).toBe(1); expect(results[2].uncached).toBe(1); // 2 -> 3 should stay cached expect(results[3].loader).toBe(results[2].loader); expect(results[3].config).toBe(1); + expect(results[3].esmConfig).toBe(1); expect(results[3].uncached).toBe(1); // 3 -> 4 should be invalidated expect(results[4].loader).toBeGreaterThan(now2); expect(results[4].config).toBeGreaterThan(now2); + expect(results[4].esmConfig).toBe(1); expect(results[4].uncached).toBe(results[4].config); // 4 -> 5 should stay cached, but uncacheable module still rebuilds expect(results[5].loader).toBe(results[4].loader); expect(results[5].config).toBe(results[4].config); expect(results[5].uncached).toBeGreaterThan(now3); + if (supportsEsm) { + // 5 -> 6 should be invalidated + expect(results[6].loader).toBeGreaterThan(now4); + expect(results[6].config).toBeGreaterThan(now4); + expect(results[6].esmConfig).toBeGreaterThan(now4); + expect(results[6].uncached).toBeGreaterThan(now4); + // 6 -> 7 should be invalidated + expect(results[7].loader).toBeGreaterThan(now5); + expect(results[7].config).toBeGreaterThan(now5); + expect(results[7].esmConfig).toBeGreaterThan(now5); + expect(results[7].esmAsyncConfig).toBeGreaterThan(now5); + expect(results[7].uncached).toBeGreaterThan(now5); + } }, 100000); }); diff --git a/test/fixtures/buildDependencies/esm-cjs-dep.js b/test/fixtures/buildDependencies/esm-cjs-dep.js new file mode 100644 index 00000000000..52d586a0e0e --- /dev/null +++ b/test/fixtures/buildDependencies/esm-cjs-dep.js @@ -0,0 +1 @@ +module.exports = require("../../js/buildDepsInput/esm-dependency"); diff --git a/test/fixtures/buildDependencies/esm-dep.mjs b/test/fixtures/buildDependencies/esm-dep.mjs new file mode 100644 index 00000000000..03e99c08cf8 --- /dev/null +++ b/test/fixtures/buildDependencies/esm-dep.mjs @@ -0,0 +1 @@ +export { default } from "./esm-cjs-dep.js"; diff --git a/test/fixtures/buildDependencies/esm.mjs b/test/fixtures/buildDependencies/esm.mjs new file mode 100644 index 00000000000..cd233797b08 --- /dev/null +++ b/test/fixtures/buildDependencies/esm.mjs @@ -0,0 +1,5 @@ +export { default } from "./esm-dep.mjs"; + +export const asyncDep = ( + await import("../../js/buildDepsInput/esm-async-dependency.mjs") +).default; diff --git a/test/fixtures/buildDependencies/index.js b/test/fixtures/buildDependencies/index.js index 9270c4df5ca..9f6bb68b7d7 100644 --- a/test/fixtures/buildDependencies/index.js +++ b/test/fixtures/buildDependencies/index.js @@ -3,5 +3,7 @@ module.exports = { loader: require("./loader!"), config: VALUE, + esmConfig: VALUE2, + esmAsyncConfig: VALUE3, uncached: require("./module") }; diff --git a/test/fixtures/buildDependencies/run.js b/test/fixtures/buildDependencies/run.js index 1e61e5ed113..d5e9df835a8 100644 --- a/test/fixtures/buildDependencies/run.js +++ b/test/fixtures/buildDependencies/run.js @@ -7,63 +7,77 @@ process.exitCode = 1; const options = JSON.parse(process.argv[3]); -const compiler = webpack( - { - mode: "development", - context: __dirname, - entry: "./index", - output: { - path: path.resolve(__dirname, "../../js/buildDeps/" + process.argv[2]), - libraryTarget: "commonjs2" - }, - plugins: [ - new webpack.DefinePlugin({ - VALUE: JSON.stringify(value), - VALUE_UNCACHEABLE: webpack.DefinePlugin.runtimeValue( - () => JSON.stringify(value), - true - ) - }) - ], - infrastructureLogging: { - level: "verbose" - }, - cache: { - type: "filesystem", - cacheDirectory: path.resolve(__dirname, "../../js/buildDepsCache"), - buildDependencies: { - config: [ - __filename, - path.resolve(__dirname, "../../../node_modules/.yarn-integrity") - ], - invalid: options.invalidBuildDepdencies ? ["should-fail-resolving"] : [] +const esm = +process.versions.modules >= 83; + +if (esm) { + import("./esm.mjs").then(module => { + run(module); + }); +} else { + run({ default: 1, asyncDep: 1 }); +} + +function run({ default: value2, asyncDep: value3 }) { + const compiler = webpack( + { + mode: "development", + context: __dirname, + entry: "./index", + output: { + path: path.resolve(__dirname, "../../js/buildDeps/" + process.argv[2]), + libraryTarget: "commonjs2" + }, + plugins: [ + new webpack.DefinePlugin({ + VALUE: JSON.stringify(value), + VALUE2: JSON.stringify(value2), + VALUE3: JSON.stringify(value3), + VALUE_UNCACHEABLE: webpack.DefinePlugin.runtimeValue( + () => JSON.stringify(value), + true + ) + }) + ], + infrastructureLogging: { + level: "verbose", + debug: /PackFile/ + }, + cache: { + type: "filesystem", + cacheDirectory: path.resolve(__dirname, "../../js/buildDepsCache"), + buildDependencies: { + config: [ + __filename, + path.resolve(__dirname, "../../../node_modules/.yarn-integrity") + ].concat(esm ? ["./esm.mjs"] : []), + invalid: options.invalidBuildDepdencies + ? ["should-fail-resolving"] + : [] + } } }, - snapshot: { - managedPaths: [path.resolve(__dirname, "../../../node_modules")] - } - }, - (err, stats) => { - if (err) { - return console.log(err); - } - if (stats.hasErrors()) { - return console.log(stats.toString({ all: false, errors: true })); - } - if (options.buildTwice) { - compiler.run((err, stats) => { - if (err) { - return console.log(err); - } - if (stats.hasErrors()) { - return console.log(stats.toString({ all: false, errors: true })); - } + (err, stats) => { + if (err) { + return console.log(err); + } + if (stats.hasErrors()) { + return console.log(stats.toString({ all: false, errors: true })); + } + if (options.buildTwice) { + compiler.run((err, stats) => { + if (err) { + return console.log(err); + } + if (stats.hasErrors()) { + return console.log(stats.toString({ all: false, errors: true })); + } + process.exitCode = 0; + console.log("OK"); + }); + } else { process.exitCode = 0; console.log("OK"); - }); - } else { - process.exitCode = 0; - console.log("OK"); + } } - } -); + ); +} From c453624a57ed3b191a5b58c73d97f41f6647c8a7 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 13 Jan 2021 20:37:45 +0100 Subject: [PATCH 2/3] store build deps correctly in cache improve debug logging for snapshotting --- lib/FileSystemInfo.js | 21 +++++++++++++-------- lib/cache/PackFileCacheStrategy.js | 10 +++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/FileSystemInfo.js b/lib/FileSystemInfo.js index 8e6733315c4..16d894df29c 100644 --- a/lib/FileSystemInfo.js +++ b/lib/FileSystemInfo.js @@ -1606,7 +1606,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting file timestamp hash combination of ${path}: ${err}` + `Error snapshotting file timestamp hash combination of ${path}: ${err.stack}` ); } jobError(); @@ -1634,7 +1634,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting file hash of ${path}: ${err}` + `Error snapshotting file hash of ${path}: ${err.stack}` ); } jobError(); @@ -1664,7 +1664,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting file timestamp of ${path}: ${err}` + `Error snapshotting file timestamp of ${path}: ${err.stack}` ); } jobError(); @@ -1700,7 +1700,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting context timestamp hash combination of ${path}: ${err}` + `Error snapshotting context timestamp hash combination of ${path}: ${err.stack}` ); } jobError(); @@ -1728,7 +1728,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting context hash of ${path}: ${err}` + `Error snapshotting context hash of ${path}: ${err.stack}` ); } jobError(); @@ -1758,7 +1758,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting context timestamp of ${path}: ${err}` + `Error snapshotting context timestamp of ${path}: ${err.stack}` ); } jobError(); @@ -1791,7 +1791,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting missing timestamp of ${path}: ${err}` + `Error snapshotting missing timestamp of ${path}: ${err.stack}` ); } jobError(); @@ -1818,7 +1818,7 @@ class FileSystemInfo { if (err) { if (this.logger) { this.logger.debug( - `Error snapshotting managed item ${path}: ${err}` + `Error snapshotting managed item ${path}: ${err.stack}` ); } jobError(); @@ -2348,6 +2348,11 @@ class FileSystemInfo { this._fileHashes.set(path, null); return callback(null, null); } + if (err.code === "ERR_FS_FILE_TOO_LARGE") { + this.logger.warn(`Ignoring ${path} for hashing as it's very large`); + this._fileHashes.set(path, "too large"); + return callback(null, "too large"); + } return callback(err); } diff --git a/lib/cache/PackFileCacheStrategy.js b/lib/cache/PackFileCacheStrategy.js index 42fefdf7d25..de9444bd8a3 100644 --- a/lib/cache/PackFileCacheStrategy.js +++ b/lib/cache/PackFileCacheStrategy.js @@ -1072,18 +1072,18 @@ class PackFileCacheStrategy { return promise.then(() => { if (reportProgress) reportProgress(0.8, "serialize pack"); this.logger.time(`store pack`); + const updatedBuildDependencies = new Set(this.buildDependencies); + for (const dep of newBuildDependencies) { + updatedBuildDependencies.add(dep); + } const content = new PackContainer( pack, this.version, this.buildSnapshot, - this.buildDependencies, + updatedBuildDependencies, this.resolveResults, this.resolveBuildDependenciesSnapshot ); - // You might think this breaks all access to the existing pack - // which are still referenced, but serializing the pack memorizes - // all data in the pack and makes it no longer need the backing file - // So it's safe to replace the pack file return this.fileSerializer .serialize(content, { filename: `${this.cacheLocation}/index.pack`, From 89aabf769672af2d324c2411cca1ed48b9c8d5d6 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 13 Jan 2021 21:16:55 +0100 Subject: [PATCH 3/3] add experimental ESM tracking based on es-module-lexer assume that other files have no dependencies --- lib/FileSystemInfo.js | 85 ++++++++++++++++++++++++++++++++++++++----- package.json | 2 + yarn.lock | 10 +++++ 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/lib/FileSystemInfo.js b/lib/FileSystemInfo.js index 16d894df29c..590929c9242 100644 --- a/lib/FileSystemInfo.js +++ b/lib/FileSystemInfo.js @@ -16,6 +16,8 @@ const makeSerializable = require("./util/makeSerializable"); /** @typedef {import("./logging/Logger").Logger} Logger */ /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */ +const supportsEsm = +process.versions.modules >= 83; + const resolveContext = createResolver({ resolveToContext: true, exportsFields: [] @@ -880,6 +882,8 @@ class FileSystemInfo { this._cachedDeprecatedFileTimestamps = undefined; this._cachedDeprecatedContextTimestamps = undefined; + this._warnAboutExperimentalEsmTracking = false; + this._statCreatedSnapshots = 0; this._statTestedSnapshotsCached = 0; this._statTestedSnapshotsNotCached = 0; @@ -1211,7 +1215,12 @@ class FileSystemInfo { break; } case RBDT_FILE_DEPENDENCIES: { - // TODO this probably doesn't work correctly with ESM dependencies + // Check for known files without dependencies + if (/\.json5?$|\.yarn-integrity$|yarn\.lock$|\.ya?ml/.test(path)) { + process.nextTick(callback); + break; + } + // Check commonjs cache for the module /** @type {NodeModule} */ const module = require.cache[path]; if (module && Array.isArray(module.children)) { @@ -1248,15 +1257,73 @@ class FileSystemInfo { }); } } + } else if (supportsEsm && /\.m?js$/.test(path)) { + if (!this._warnAboutExperimentalEsmTracking) { + this.logger.info( + "Node.js doesn't offer a (nice) way to introspect the ESM dependency graph yet.\n" + + "Until a full solution is available webpack uses an experimental ESM tracking based on parsing.\n" + + "As best effort webpack parses the ESM files to guess dependencies. But this can lead to expensive and incorrect tracking." + ); + this._warnAboutExperimentalEsmTracking = true; + } + const lexer = require("es-module-lexer"); + lexer.init.then(() => { + this.fs.readFile(path, (err, content) => { + if (err) return callback(err); + try { + const context = dirname(this.fs, path); + const source = content.toString(); + const [imports] = lexer.parse(source); + for (const imp of imports) { + try { + let dependency; + if (imp.d === -1) { + // import ... from "..." + dependency = JSON.parse( + source.substring(imp.s - 1, imp.e + 1) + ); + } else if (imp.d > -1) { + // import() + let expr = source.substring(imp.s, imp.e).trim(); + if (expr[0] === "'") + expr = `"${expr + .slice(1, -1) + .replace(/"/g, '\\"')}"`; + dependency = JSON.parse(expr); + } else { + // e.g. import.meta + continue; + } + queue.push({ + type: RBDT_RESOLVE_FILE, + context, + path: dependency + }); + } catch (e) { + this.logger.warn( + `Parsing of ${path} for build dependencies failed at 'import(${source.substring( + imp.s, + imp.e + )})'.\n` + + "Build dependencies behind this expression are ignored and might cause incorrect cache invalidation." + ); + this.logger.debug(e.stack); + } + } + } catch (e) { + this.logger.warn( + `Parsing of ${path} for build dependencies failed and all dependencies of this file are ignored, which might cause incorrect cache invalidation..` + ); + this.logger.debug(e.stack); + } + process.nextTick(callback); + }); + }, callback); + break; } else { - // Unable to get dependencies from module system - // This may be because of an incomplete require.cache implementation like in jest - // Assume requires stay in directory and add the whole directory - const directory = dirname(this.fs, path); - queue.push({ - type: RBDT_DIRECTORY, - path: directory - }); + this.logger.log( + `Assuming ${path} has no dependencies as we were unable to assign it to any module system.` + ); } process.nextTick(callback); break; diff --git a/package.json b/package.json index 5c03f8a237d..18a49a359c0 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.7.0", + "es-module-lexer": "^0.3.26", "eslint-scope": "^5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", @@ -37,6 +38,7 @@ "devDependencies": { "@babel/core": "^7.11.1", "@babel/preset-react": "^7.10.4", + "@types/es-module-lexer": "^0.3.0", "@types/jest": "^26.0.15", "@types/node": "^14.14.10", "babel-loader": "^8.1.0", diff --git a/yarn.lock b/yarn.lock index 1b3394a1b18..83b4231b305 100644 --- a/yarn.lock +++ b/yarn.lock @@ -836,6 +836,11 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== +"@types/es-module-lexer@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@types/es-module-lexer/-/es-module-lexer-0.3.0.tgz#9fee3f19f64e6b3f999eeb3a70bd177a4d57a6cb" + integrity sha512-XI3MGSejUQIJ3wzY0i5IHy5J3eb36M/ytgG8jIOssP08ovtRPcjpjXQqrx51AHBNBOisTS/NQNWJitI17+EwzQ== + "@types/eslint-scope@^3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86" @@ -2345,6 +2350,11 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +es-module-lexer@^0.3.26: + version "0.3.26" + resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.3.26.tgz#7b507044e97d5b03b01d4392c74ffeb9c177a83b" + integrity sha512-Va0Q/xqtrss45hWzP8CZJwzGSZJjDM5/MJRE3IXXnUCcVLElR9BRaE9F62BopysASyc4nM3uwhSW7FFB9nlWAA== + es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@^0.10.53, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46: version "0.10.53" resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1"