Merge pull request #14728 from markjm/markjm/context-timestamps

perf: upgrade watchpack & split timestamp by file/dependency & only call collectTimeInfoEntries once per invalid

lib/WatchIgnorePlugin.js

@@ -87,7 +87,20 @@ class IgnoringWatchFileSystem {
 					fileTimestamps.set(path, IGNORE_TIME_ENTRY);
 				}
 				return fileTimestamps;
-			}
+			},
+			getInfo:
+				watcher.getInfo &&
+				(() => {
+					const info = watcher.getInfo();
+					const { fileTimeInfoEntries, contextTimeInfoEntries } = info;
+					for (const path of ignoredFiles) {
+						fileTimeInfoEntries.set(path, IGNORE_TIME_ENTRY);
+					}
+					for (const path of ignoredDirs) {
+						contextTimeInfoEntries.set(path, IGNORE_TIME_ENTRY);
+					}
+					return info;
+				})
 		};
 	}
 }

lib/Watching.js

@@ -109,30 +109,44 @@ class Watching {
 			this.lastWatcherStartTime = Date.now();
 		}
 		this.compiler.fsStartTime = Date.now();
-		this._mergeWithCollected(
-			changedFiles ||
-				(this.pausedWatcher &&
+		if (
+			changedFiles &&
+			removedFiles &&
+			fileTimeInfoEntries &&
+			contextTimeInfoEntries
+		) {
+			this._mergeWithCollected(changedFiles, removedFiles);
+			this.compiler.fileTimestamps = fileTimeInfoEntries;
+			this.compiler.contextTimestamps = contextTimeInfoEntries;
+		} else if (this.pausedWatcher) {
+			if (this.pausedWatcher.getInfo) {
+				const {
+					changes,
+					removals,
+					fileTimeInfoEntries,
+					contextTimeInfoEntries
+				} = this.pausedWatcher.getInfo();
+				this._mergeWithCollected(changes, removals);
+				this.compiler.fileTimestamps = fileTimeInfoEntries;
+				this.compiler.contextTimestamps = contextTimeInfoEntries;
+			} else {
+				this._mergeWithCollected(
 					this.pausedWatcher.getAggregatedChanges &&
-					this.pausedWatcher.getAggregatedChanges()),
-			(this.compiler.removedFiles =
-				removedFiles ||
-				(this.pausedWatcher &&
+						this.pausedWatcher.getAggregatedChanges(),
 					this.pausedWatcher.getAggregatedRemovals &&
-					this.pausedWatcher.getAggregatedRemovals()))
-		);
-
+						this.pausedWatcher.getAggregatedRemovals()
+				);
+				this.compiler.fileTimestamps =
+					this.pausedWatcher.getFileTimeInfoEntries();
+				this.compiler.contextTimestamps =
+					this.pausedWatcher.getContextTimeInfoEntries();
+			}
+		}
 		this.compiler.modifiedFiles = this._collectedChangedFiles;
 		this._collectedChangedFiles = undefined;
 		this.compiler.removedFiles = this._collectedRemovedFiles;
 		this._collectedRemovedFiles = undefined;
 
-		this.compiler.fileTimestamps =
-			fileTimeInfoEntries ||
-			(this.pausedWatcher && this.pausedWatcher.getFileTimeInfoEntries());
-		this.compiler.contextTimestamps =
-			contextTimeInfoEntries ||
-			(this.pausedWatcher && this.pausedWatcher.getContextTimeInfoEntries());
-
 		const run = () => {
 			if (this.compiler.idle) {
 				return this.compiler.cache.endIdle(err => {

lib/node/NodeWatchFileSystem.js

@@ -5,6 +5,7 @@
 
 "use strict";
 
+const util = require("util");
 const Watchpack = require("watchpack");
 
 /** @typedef {import("../../declarations/WebpackOptions").WatchOptions} WatchOptions */
@@ -68,7 +69,22 @@ class NodeWatchFileSystem {
 		if (callbackUndelayed) {
 			this.watcher.once("change", callbackUndelayed);
 		}
+
+		const fetchTimeInfo = () => {
+			const fileTimeInfoEntries = new Map();
+			const contextTimeInfoEntries = new Map();
+			if (this.watcher) {
+				this.watcher.collectTimeInfoEntries(
+					fileTimeInfoEntries,
+					contextTimeInfoEntries
+				);
+			}
+			return { fileTimeInfoEntries, contextTimeInfoEntries };
+		};
 		this.watcher.once("aggregated", (changes, removals) => {
+			// pause emitting events (avoids clearing aggregated changes and removals on timeout)
+			this.watcher.pause();
+
 			if (this.inputFileSystem && this.inputFileSystem.purge) {
 				const fs = this.inputFileSystem;
 				for (const item of changes) {
@@ -78,8 +94,14 @@ class NodeWatchFileSystem {
 					fs.purge(item);
 				}
 			}
-			const times = this.watcher.getTimeInfoEntries();
-			callback(null, times, times, changes, removals);
+			const { fileTimeInfoEntries, contextTimeInfoEntries } = fetchTimeInfo();
+			callback(
+				null,
+				fileTimeInfoEntries,
+				contextTimeInfoEntries,
+				changes,
+				removals
+			);
 		});
 
 		this.watcher.watch({ files, directories, missing, startTime });
@@ -99,39 +121,71 @@ class NodeWatchFileSystem {
 					this.watcher.pause();
 				}
 			},
-			getAggregatedRemovals: () => {
-				const items = this.watcher && this.watcher.aggregatedRemovals;
-				if (items && this.inputFileSystem && this.inputFileSystem.purge) {
-					const fs = this.inputFileSystem;
-					for (const item of items) {
-						fs.purge(item);
+			getAggregatedRemovals: util.deprecate(
+				() => {
+					const items = this.watcher && this.watcher.aggregatedRemovals;
+					if (items && this.inputFileSystem && this.inputFileSystem.purge) {
+						const fs = this.inputFileSystem;
+						for (const item of items) {
+							fs.purge(item);
+						}
 					}
-				}
-				return items;
-			},
-			getAggregatedChanges: () => {
-				const items = this.watcher && this.watcher.aggregatedChanges;
-				if (items && this.inputFileSystem && this.inputFileSystem.purge) {
+					return items;
+				},
+				"Watcher.getAggregatedRemovals is deprecated in favor of Watcher.getInfo since that's more performant.",
+				"DEP_WEBPACK_WATCHER_GET_AGGREGATED_REMOVALS"
+			),
+			getAggregatedChanges: util.deprecate(
+				() => {
+					const items = this.watcher && this.watcher.aggregatedChanges;
+					if (items && this.inputFileSystem && this.inputFileSystem.purge) {
+						const fs = this.inputFileSystem;
+						for (const item of items) {
+							fs.purge(item);
+						}
+					}
+					return items;
+				},
+				"Watcher.getAggregatedChanges is deprecated in favor of Watcher.getInfo since that's more performant.",
+				"DEP_WEBPACK_WATCHER_GET_AGGREGATED_CHANGES"
+			),
+			getFileTimeInfoEntries: util.deprecate(
+				() => {
+					return fetchTimeInfo().fileTimeInfoEntries;
+				},
+				"Watcher.getFileTimeInfoEntries is deprecated in favor of Watcher.getInfo since that's more performant.",
+				"DEP_WEBPACK_WATCHER_FILE_TIME_INFO_ENTRIES"
+			),
+			getContextTimeInfoEntries: util.deprecate(
+				() => {
+					return fetchTimeInfo().contextTimeInfoEntries;
+				},
+				"Watcher.getContextTimeInfoEntries is deprecated in favor of Watcher.getInfo since that's more performant.",
+				"DEP_WEBPACK_WATCHER_CONTEXT_TIME_INFO_ENTRIES"
+			),
+			getInfo: () => {
+				const removals = this.watcher && this.watcher.aggregatedRemovals;
+				const changes = this.watcher && this.watcher.aggregatedChanges;
+				if (this.inputFileSystem && this.inputFileSystem.purge) {
 					const fs = this.inputFileSystem;
-					for (const item of items) {
-						fs.purge(item);
+					if (removals) {
+						for (const item of removals) {
+							fs.purge(item);
+						}
+					}
+					if (changes) {
+						for (const item of changes) {
+							fs.purge(item);
+						}
 					}
 				}
-				return items;
-			},
-			getFileTimeInfoEntries: () => {
-				if (this.watcher) {
-					return this.watcher.getTimeInfoEntries();
-				} else {
-					return new Map();
-				}
-			},
-			getContextTimeInfoEntries: () => {
-				if (this.watcher) {
-					return this.watcher.getTimeInfoEntries();
-				} else {
-					return new Map();
-				}
+				const { fileTimeInfoEntries, contextTimeInfoEntries } = fetchTimeInfo();
+				return {
+					changes,
+					removals,
+					fileTimeInfoEntries,
+					contextTimeInfoEntries
+				};
 			}
 		};
 	}

lib/util/fs.js

@@ -61,6 +61,15 @@ const path = require("path");
 /** @typedef {function((NodeJS.ErrnoException | Error | null)=, any=): void} ReadJsonCallback */
 /** @typedef {function((NodeJS.ErrnoException | Error | null)=, IStats|string=): void} LstatReadlinkAbsoluteCallback */
 
+/**
+ * @typedef {Object} WatcherInfo
+ * @property {Set<string>} changes get current aggregated changes that have not yet send to callback
+ * @property {Set<string>} removals get current aggregated removals that have not yet send to callback
+ * @property {Map<string, FileSystemInfoEntry | "ignore">} fileTimeInfoEntries get info about files
+ * @property {Map<string, FileSystemInfoEntry | "ignore">} contextTimeInfoEntries get info about directories
+ */
+
+// TODO webpack 6 deprecate missing getInfo
 /**
  * @typedef {Object} Watcher
  * @property {function(): void} close closes the watcher and all underlying file watchers
@@ -69,6 +78,7 @@ const path = require("path");
  * @property {function(): Set<string>=} getAggregatedRemovals get current aggregated removals that have not yet send to callback
  * @property {function(): Map<string, FileSystemInfoEntry | "ignore">} getFileTimeInfoEntries get info about files
  * @property {function(): Map<string, FileSystemInfoEntry | "ignore">} getContextTimeInfoEntries get info about directories
+ * @property {function(): WatcherInfo=} getInfo get info about timestamps and changes
  */
 
 /**

package.json

@@ -27,7 +27,7 @@
     "schema-utils": "^3.1.0",
     "tapable": "^2.1.1",
     "terser-webpack-plugin": "^5.1.3",
-    "watchpack": "^2.2.0",
+    "watchpack": "^2.3.0",
     "webpack-sources": "^3.2.2"
   },
   "peerDependenciesMeta": {

types.d.ts

@@ -11741,6 +11741,32 @@ declare interface Watcher {
 	 * get info about directories
 	 */
 	getContextTimeInfoEntries: () => Map<string, FileSystemInfoEntry | "ignore">;
+
+	/**
+	 * get info about timestamps and changes
+	 */
+	getInfo?: () => WatcherInfo;
+}
+declare interface WatcherInfo {
+	/**
+	 * get current aggregated changes that have not yet send to callback
+	 */
+	changes: Set<string>;
+
+	/**
+	 * get current aggregated removals that have not yet send to callback
+	 */
+	removals: Set<string>;
+
+	/**
+	 * get info about files
+	 */
+	fileTimeInfoEntries: Map<string, FileSystemInfoEntry | "ignore">;
+
+	/**
+	 * get info about directories
+	 */
+	contextTimeInfoEntries: Map<string, FileSystemInfoEntry | "ignore">;
 }
 declare abstract class Watching {
 	startTime: null | number;

yarn.lock

@@ -6074,10 +6074,10 @@ wast-loader@^1.11.0:
   dependencies:
     wabt "1.0.0-nightly.20180421"
 
-watchpack@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.2.0.tgz#47d78f5415fe550ecd740f99fe2882323a58b1ce"
-  integrity sha512-up4YAn/XHgZHIxFBVCdlMiWDj6WaLKpwVeGQk2I5thdYxF/KmF0aaz6TfJZ/hfl1h/XlcDr7k1KH7ThDagpFaA==
+watchpack@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.3.0.tgz#a41bca3da6afaff31e92a433f4c856a0c25ea0c4"
+  integrity sha512-MnN0Q1OsvB/GGHETrFeZPQaOelWh/7O+EiFlj8sM9GPjtQkis7k01aAxrg/18kTfoIVcLL+haEVFlXDaSRwKRw==
   dependencies:
     glob-to-regexp "^0.4.1"
     graceful-fs "^4.1.2"