From c2f57247a8ef7057d17908ed2bcefa66c8b55362 Mon Sep 17 00:00:00 2001 From: Mark Molinaro Date: Wed, 17 Nov 2021 11:43:54 -0800 Subject: [PATCH 1/4] perf: correctly split timestamp by file/dependency and only call getTimeInfoEntries once --- lib/node/NodeWatchFileSystem.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/node/NodeWatchFileSystem.js b/lib/node/NodeWatchFileSystem.js index 67ef918b8a3..d774dadf06b 100644 --- a/lib/node/NodeWatchFileSystem.js +++ b/lib/node/NodeWatchFileSystem.js @@ -68,6 +68,8 @@ class NodeWatchFileSystem { if (callbackUndelayed) { this.watcher.once("change", callbackUndelayed); } + + let fileMap, directoryMap; this.watcher.once("aggregated", (changes, removals) => { if (this.inputFileSystem && this.inputFileSystem.purge) { const fs = this.inputFileSystem; @@ -78,8 +80,10 @@ class NodeWatchFileSystem { fs.purge(item); } } - const times = this.watcher.getTimeInfoEntries(); - callback(null, times, times, changes, removals); + fileMap = new Map(); + directoryMap = new Map(); + this.watcher.getTimeInfoEntries(fileMap, directoryMap); + callback(null, fileMap, directoryMap, changes, removals); }); this.watcher.watch({ files, directories, missing, startTime }); @@ -120,18 +124,20 @@ class NodeWatchFileSystem { return items; }, getFileTimeInfoEntries: () => { + if (fileMap) return fileMap; if (this.watcher) { - return this.watcher.getTimeInfoEntries(); - } else { - return new Map(); + this.watcher.getTimeInfoEntries(fileMap, directoryMap); + return fileMap; } + return new Map(); }, getContextTimeInfoEntries: () => { + if (directoryMap) return directoryMap; if (this.watcher) { - return this.watcher.getTimeInfoEntries(); - } else { - return new Map(); + this.watcher.getTimeInfoEntries(fileMap, directoryMap); + return directoryMap; } + return new Map(); } }; } From 5601c5618599d035340567a04221c0c4f47c7dce Mon Sep 17 00:00:00 2001 From: Mark Molinaro Date: Wed, 17 Nov 2021 11:46:43 -0800 Subject: [PATCH 2/4] use fresh maps when not available --- lib/node/NodeWatchFileSystem.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/node/NodeWatchFileSystem.js b/lib/node/NodeWatchFileSystem.js index d774dadf06b..a081769c3bb 100644 --- a/lib/node/NodeWatchFileSystem.js +++ b/lib/node/NodeWatchFileSystem.js @@ -126,7 +126,10 @@ class NodeWatchFileSystem { getFileTimeInfoEntries: () => { if (fileMap) return fileMap; if (this.watcher) { - this.watcher.getTimeInfoEntries(fileMap, directoryMap); + this.watcher.getTimeInfoEntries( + (fileMap = new Map()), + (directoryMap = new Map()) + ); return fileMap; } return new Map(); @@ -134,7 +137,10 @@ class NodeWatchFileSystem { getContextTimeInfoEntries: () => { if (directoryMap) return directoryMap; if (this.watcher) { - this.watcher.getTimeInfoEntries(fileMap, directoryMap); + this.watcher.getTimeInfoEntries( + (fileMap = new Map()), + (directoryMap = new Map()) + ); return directoryMap; } return new Map(); From 8867bc18ce967a85dfe9c53351c31b0043a3d89b Mon Sep 17 00:00:00 2001 From: Mark Molinaro Date: Wed, 24 Nov 2021 15:31:14 -0800 Subject: [PATCH 3/4] use new watchpack collectTimeInfoEntries --- lib/node/NodeWatchFileSystem.js | 28 ++++++++++++++++++++-------- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/node/NodeWatchFileSystem.js b/lib/node/NodeWatchFileSystem.js index 67ef918b8a3..99f1b8de12b 100644 --- a/lib/node/NodeWatchFileSystem.js +++ b/lib/node/NodeWatchFileSystem.js @@ -68,6 +68,8 @@ class NodeWatchFileSystem { if (callbackUndelayed) { this.watcher.once("change", callbackUndelayed); } + + let fileMap, directoryMap; this.watcher.once("aggregated", (changes, removals) => { if (this.inputFileSystem && this.inputFileSystem.purge) { const fs = this.inputFileSystem; @@ -78,8 +80,10 @@ class NodeWatchFileSystem { fs.purge(item); } } - const times = this.watcher.getTimeInfoEntries(); - callback(null, times, times, changes, removals); + fileMap = new Map(); + directoryMap = new Map(); + this.watcher.collectTimeInfoEntries(fileMap, directoryMap); + callback(null, fileMap, directoryMap, changes, removals); }); this.watcher.watch({ files, directories, missing, startTime }); @@ -120,18 +124,26 @@ class NodeWatchFileSystem { return items; }, getFileTimeInfoEntries: () => { + if (fileMap) return fileMap; if (this.watcher) { - return this.watcher.getTimeInfoEntries(); - } else { - return new Map(); + this.watcher.collectTimeInfoEntries( + (fileMap = new Map()), + (directoryMap = new Map()) + ); + return fileMap; } + return new Map(); }, getContextTimeInfoEntries: () => { + if (directoryMap) return directoryMap; if (this.watcher) { - return this.watcher.getTimeInfoEntries(); - } else { - return new Map(); + this.watcher.collectTimeInfoEntries( + (fileMap = new Map()), + (directoryMap = new Map()) + ); + return directoryMap; } + return new Map(); } }; } diff --git a/package.json b/package.json index 03f49ada01a..6453eafca3c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "schema-utils": "^3.1.0", "tapable": "^2.1.1", "terser-webpack-plugin": "^5.1.3", - "watchpack": "^2.2.0", + "watchpack": "^2.3.0", "webpack-sources": "^3.2.2" }, "peerDependenciesMeta": { diff --git a/yarn.lock b/yarn.lock index 3ac71f65989..379bc557949 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6074,10 +6074,10 @@ wast-loader@^1.11.0: dependencies: wabt "1.0.0-nightly.20180421" -watchpack@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.2.0.tgz#47d78f5415fe550ecd740f99fe2882323a58b1ce" - integrity sha512-up4YAn/XHgZHIxFBVCdlMiWDj6WaLKpwVeGQk2I5thdYxF/KmF0aaz6TfJZ/hfl1h/XlcDr7k1KH7ThDagpFaA== +watchpack@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.3.0.tgz#a41bca3da6afaff31e92a433f4c856a0c25ea0c4" + integrity sha512-MnN0Q1OsvB/GGHETrFeZPQaOelWh/7O+EiFlj8sM9GPjtQkis7k01aAxrg/18kTfoIVcLL+haEVFlXDaSRwKRw== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" From 7025319eea435c366049522791065e6429daf430 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 25 Nov 2021 09:27:04 +0100 Subject: [PATCH 4/4] upgrade watchpack use new collectTimeInfoEntries method from watchpack add more efficient Watcher.getInfo method --- lib/WatchIgnorePlugin.js | 15 +++- lib/Watching.js | 48 +++++++----- lib/node/NodeWatchFileSystem.js | 126 +++++++++++++++++++++----------- lib/util/fs.js | 10 +++ package.json | 2 +- types.d.ts | 26 +++++++ yarn.lock | 8 +- 7 files changed, 170 insertions(+), 65 deletions(-) diff --git a/lib/WatchIgnorePlugin.js b/lib/WatchIgnorePlugin.js index 5b5b26cd4db..52cde68284e 100644 --- a/lib/WatchIgnorePlugin.js +++ b/lib/WatchIgnorePlugin.js @@ -87,7 +87,20 @@ class IgnoringWatchFileSystem { fileTimestamps.set(path, IGNORE_TIME_ENTRY); } return fileTimestamps; - } + }, + getInfo: + watcher.getInfo && + (() => { + const info = watcher.getInfo(); + const { fileTimeInfoEntries, contextTimeInfoEntries } = info; + for (const path of ignoredFiles) { + fileTimeInfoEntries.set(path, IGNORE_TIME_ENTRY); + } + for (const path of ignoredDirs) { + contextTimeInfoEntries.set(path, IGNORE_TIME_ENTRY); + } + return info; + }) }; } } diff --git a/lib/Watching.js b/lib/Watching.js index fc6c176b714..f92b55119e7 100644 --- a/lib/Watching.js +++ b/lib/Watching.js @@ -109,30 +109,44 @@ class Watching { this.lastWatcherStartTime = Date.now(); } this.compiler.fsStartTime = Date.now(); - this._mergeWithCollected( - changedFiles || - (this.pausedWatcher && + if ( + changedFiles && + removedFiles && + fileTimeInfoEntries && + contextTimeInfoEntries + ) { + this._mergeWithCollected(changedFiles, removedFiles); + this.compiler.fileTimestamps = fileTimeInfoEntries; + this.compiler.contextTimestamps = contextTimeInfoEntries; + } else if (this.pausedWatcher) { + if (this.pausedWatcher.getInfo) { + const { + changes, + removals, + fileTimeInfoEntries, + contextTimeInfoEntries + } = this.pausedWatcher.getInfo(); + this._mergeWithCollected(changes, removals); + this.compiler.fileTimestamps = fileTimeInfoEntries; + this.compiler.contextTimestamps = contextTimeInfoEntries; + } else { + this._mergeWithCollected( this.pausedWatcher.getAggregatedChanges && - this.pausedWatcher.getAggregatedChanges()), - (this.compiler.removedFiles = - removedFiles || - (this.pausedWatcher && + this.pausedWatcher.getAggregatedChanges(), this.pausedWatcher.getAggregatedRemovals && - this.pausedWatcher.getAggregatedRemovals())) - ); - + this.pausedWatcher.getAggregatedRemovals() + ); + this.compiler.fileTimestamps = + this.pausedWatcher.getFileTimeInfoEntries(); + this.compiler.contextTimestamps = + this.pausedWatcher.getContextTimeInfoEntries(); + } + } this.compiler.modifiedFiles = this._collectedChangedFiles; this._collectedChangedFiles = undefined; this.compiler.removedFiles = this._collectedRemovedFiles; this._collectedRemovedFiles = undefined; - this.compiler.fileTimestamps = - fileTimeInfoEntries || - (this.pausedWatcher && this.pausedWatcher.getFileTimeInfoEntries()); - this.compiler.contextTimestamps = - contextTimeInfoEntries || - (this.pausedWatcher && this.pausedWatcher.getContextTimeInfoEntries()); - const run = () => { if (this.compiler.idle) { return this.compiler.cache.endIdle(err => { diff --git a/lib/node/NodeWatchFileSystem.js b/lib/node/NodeWatchFileSystem.js index a081769c3bb..0cf5e820af5 100644 --- a/lib/node/NodeWatchFileSystem.js +++ b/lib/node/NodeWatchFileSystem.js @@ -5,6 +5,7 @@ "use strict"; +const util = require("util"); const Watchpack = require("watchpack"); /** @typedef {import("../../declarations/WebpackOptions").WatchOptions} WatchOptions */ @@ -69,8 +70,21 @@ class NodeWatchFileSystem { this.watcher.once("change", callbackUndelayed); } - let fileMap, directoryMap; + const fetchTimeInfo = () => { + const fileTimeInfoEntries = new Map(); + const contextTimeInfoEntries = new Map(); + if (this.watcher) { + this.watcher.collectTimeInfoEntries( + fileTimeInfoEntries, + contextTimeInfoEntries + ); + } + return { fileTimeInfoEntries, contextTimeInfoEntries }; + }; this.watcher.once("aggregated", (changes, removals) => { + // pause emitting events (avoids clearing aggregated changes and removals on timeout) + this.watcher.pause(); + if (this.inputFileSystem && this.inputFileSystem.purge) { const fs = this.inputFileSystem; for (const item of changes) { @@ -80,10 +94,14 @@ class NodeWatchFileSystem { fs.purge(item); } } - fileMap = new Map(); - directoryMap = new Map(); - this.watcher.getTimeInfoEntries(fileMap, directoryMap); - callback(null, fileMap, directoryMap, changes, removals); + const { fileTimeInfoEntries, contextTimeInfoEntries } = fetchTimeInfo(); + callback( + null, + fileTimeInfoEntries, + contextTimeInfoEntries, + changes, + removals + ); }); this.watcher.watch({ files, directories, missing, startTime }); @@ -103,47 +121,71 @@ class NodeWatchFileSystem { this.watcher.pause(); } }, - getAggregatedRemovals: () => { - const items = this.watcher && this.watcher.aggregatedRemovals; - if (items && this.inputFileSystem && this.inputFileSystem.purge) { - const fs = this.inputFileSystem; - for (const item of items) { - fs.purge(item); + getAggregatedRemovals: util.deprecate( + () => { + const items = this.watcher && this.watcher.aggregatedRemovals; + if (items && this.inputFileSystem && this.inputFileSystem.purge) { + const fs = this.inputFileSystem; + for (const item of items) { + fs.purge(item); + } } - } - return items; - }, - getAggregatedChanges: () => { - const items = this.watcher && this.watcher.aggregatedChanges; - if (items && this.inputFileSystem && this.inputFileSystem.purge) { + return items; + }, + "Watcher.getAggregatedRemovals is deprecated in favor of Watcher.getInfo since that's more performant.", + "DEP_WEBPACK_WATCHER_GET_AGGREGATED_REMOVALS" + ), + getAggregatedChanges: util.deprecate( + () => { + const items = this.watcher && this.watcher.aggregatedChanges; + if (items && this.inputFileSystem && this.inputFileSystem.purge) { + const fs = this.inputFileSystem; + for (const item of items) { + fs.purge(item); + } + } + return items; + }, + "Watcher.getAggregatedChanges is deprecated in favor of Watcher.getInfo since that's more performant.", + "DEP_WEBPACK_WATCHER_GET_AGGREGATED_CHANGES" + ), + getFileTimeInfoEntries: util.deprecate( + () => { + return fetchTimeInfo().fileTimeInfoEntries; + }, + "Watcher.getFileTimeInfoEntries is deprecated in favor of Watcher.getInfo since that's more performant.", + "DEP_WEBPACK_WATCHER_FILE_TIME_INFO_ENTRIES" + ), + getContextTimeInfoEntries: util.deprecate( + () => { + return fetchTimeInfo().contextTimeInfoEntries; + }, + "Watcher.getContextTimeInfoEntries is deprecated in favor of Watcher.getInfo since that's more performant.", + "DEP_WEBPACK_WATCHER_CONTEXT_TIME_INFO_ENTRIES" + ), + getInfo: () => { + const removals = this.watcher && this.watcher.aggregatedRemovals; + const changes = this.watcher && this.watcher.aggregatedChanges; + if (this.inputFileSystem && this.inputFileSystem.purge) { const fs = this.inputFileSystem; - for (const item of items) { - fs.purge(item); + if (removals) { + for (const item of removals) { + fs.purge(item); + } + } + if (changes) { + for (const item of changes) { + fs.purge(item); + } } } - return items; - }, - getFileTimeInfoEntries: () => { - if (fileMap) return fileMap; - if (this.watcher) { - this.watcher.getTimeInfoEntries( - (fileMap = new Map()), - (directoryMap = new Map()) - ); - return fileMap; - } - return new Map(); - }, - getContextTimeInfoEntries: () => { - if (directoryMap) return directoryMap; - if (this.watcher) { - this.watcher.getTimeInfoEntries( - (fileMap = new Map()), - (directoryMap = new Map()) - ); - return directoryMap; - } - return new Map(); + const { fileTimeInfoEntries, contextTimeInfoEntries } = fetchTimeInfo(); + return { + changes, + removals, + fileTimeInfoEntries, + contextTimeInfoEntries + }; } }; } diff --git a/lib/util/fs.js b/lib/util/fs.js index d93d7806fe7..bcbf571269e 100644 --- a/lib/util/fs.js +++ b/lib/util/fs.js @@ -61,6 +61,15 @@ const path = require("path"); /** @typedef {function((NodeJS.ErrnoException | Error | null)=, any=): void} ReadJsonCallback */ /** @typedef {function((NodeJS.ErrnoException | Error | null)=, IStats|string=): void} LstatReadlinkAbsoluteCallback */ +/** + * @typedef {Object} WatcherInfo + * @property {Set} changes get current aggregated changes that have not yet send to callback + * @property {Set} removals get current aggregated removals that have not yet send to callback + * @property {Map} fileTimeInfoEntries get info about files + * @property {Map} contextTimeInfoEntries get info about directories + */ + +// TODO webpack 6 deprecate missing getInfo /** * @typedef {Object} Watcher * @property {function(): void} close closes the watcher and all underlying file watchers @@ -69,6 +78,7 @@ const path = require("path"); * @property {function(): Set=} getAggregatedRemovals get current aggregated removals that have not yet send to callback * @property {function(): Map} getFileTimeInfoEntries get info about files * @property {function(): Map} getContextTimeInfoEntries get info about directories + * @property {function(): WatcherInfo=} getInfo get info about timestamps and changes */ /** diff --git a/package.json b/package.json index 03f49ada01a..6453eafca3c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "schema-utils": "^3.1.0", "tapable": "^2.1.1", "terser-webpack-plugin": "^5.1.3", - "watchpack": "^2.2.0", + "watchpack": "^2.3.0", "webpack-sources": "^3.2.2" }, "peerDependenciesMeta": { diff --git a/types.d.ts b/types.d.ts index f632e85258d..6ad09b91f7c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -11741,6 +11741,32 @@ declare interface Watcher { * get info about directories */ getContextTimeInfoEntries: () => Map; + + /** + * get info about timestamps and changes + */ + getInfo?: () => WatcherInfo; +} +declare interface WatcherInfo { + /** + * get current aggregated changes that have not yet send to callback + */ + changes: Set; + + /** + * get current aggregated removals that have not yet send to callback + */ + removals: Set; + + /** + * get info about files + */ + fileTimeInfoEntries: Map; + + /** + * get info about directories + */ + contextTimeInfoEntries: Map; } declare abstract class Watching { startTime: null | number; diff --git a/yarn.lock b/yarn.lock index 3ac71f65989..379bc557949 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6074,10 +6074,10 @@ wast-loader@^1.11.0: dependencies: wabt "1.0.0-nightly.20180421" -watchpack@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.2.0.tgz#47d78f5415fe550ecd740f99fe2882323a58b1ce" - integrity sha512-up4YAn/XHgZHIxFBVCdlMiWDj6WaLKpwVeGQk2I5thdYxF/KmF0aaz6TfJZ/hfl1h/XlcDr7k1KH7ThDagpFaA== +watchpack@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.3.0.tgz#a41bca3da6afaff31e92a433f4c856a0c25ea0c4" + integrity sha512-MnN0Q1OsvB/GGHETrFeZPQaOelWh/7O+EiFlj8sM9GPjtQkis7k01aAxrg/18kTfoIVcLL+haEVFlXDaSRwKRw== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2"