Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

improve watcher update performance #208

Merged
merged 1 commit into from Nov 24, 2021
Merged

improve watcher update performance #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
187 changes: 125 additions & 62 deletions lib/watchpack.js
View file
Open in desktop
Expand Up @@ -15,6 +15,16 @@ let EXISTANCE_ONLY_TIME_ENTRY; // lazy required
const EMPTY_ARRAY = [];
const EMPTY_OPTIONS = {};

function addWatchpackWatchersToSet(watchers, set) {
for (const ww of watchers) {
const w = ww.watcher;
if (!set.has(w.directoryWatcher)) {
set.add(w.directoryWatcher);
addWatchersToSet(w.directoryWatcher.directories.values(), set);
}
}
}

function addWatchersToSet(watchers, set) {
for (const w of watchers) {
if (w !== true && !set.has(w.directoryWatcher)) {
Expand Down Expand Up @@ -62,6 +72,85 @@ const cachedNormalizeOptions = options => {
return normalized;
};

class WatchpackFileWatcher {
constructor(watchpack, watcher, files) {
if (!watcher) throw new Error();
this.files = Array.isArray(files) ? files : [files];
this.watcher = watcher;
watcher.on("initial-missing", type => {
for (const file of this.files) {
if (!watchpack._missing.has(file))
watchpack._onRemove(file, file, type);
}
});
watcher.on("change", (mtime, type) => {
for (const file of this.files) {
watchpack._onChange(file, mtime, file, type);
}
});
watcher.on("remove", type => {
for (const file of this.files) {
watchpack._onRemove(file, file, type);
}
});
}

update(files) {
if (!Array.isArray(files)) {
if (this.files.length !== 1) {
this.files = [files];
} else if (this.files[0] !== files) {
this.files[0] = files;
}
} else {
this.files = files;
}
}

close() {
this.watcher.close();
}
}

class WatchpackDirectoryWatcher {
constructor(watchpack, watcher, directories) {
if (!watcher) throw new Error();
this.directories = Array.isArray(directories) ? directories : [directories];
this.watcher = watcher;
watcher.on("initial-missing", type => {
for (const item of this.directories) {
watchpack._onRemove(item, item, type);
}
});
watcher.on("change", (file, mtime, type) => {
for (const item of this.directories) {
watchpack._onChange(item, mtime, file, type);
}
});
watcher.on("remove", type => {
for (const item of this.directories) {
watchpack._onRemove(item, item, type);
}
});
}

update(directories) {
if (!Array.isArray(directories)) {
if (this.directories.length !== 1) {
this.directories = [directories];
} else if (this.directories[0] !== directories) {
this.directories[0] = directories;
}
} else {
this.directories = directories;
}
}

close() {
this.watcher.close();
}
}

class Watchpack extends EventEmitter {
constructor(options) {
super();
Expand All @@ -75,6 +164,7 @@ class Watchpack extends EventEmitter {
this.watcherManager = getWatcherManager(this.watcherOptions);
this.fileWatchers = new Map();
this.directoryWatchers = new Map();
this._missing = new Set();
this.startTime = undefined;
this.paused = false;
this.aggregatedChanges = new Set();
Expand All @@ -99,18 +189,20 @@ class Watchpack extends EventEmitter {
startTime = arg3;
}
this.paused = false;
const oldFileWatchers = this.fileWatchers;
const oldDirectoryWatchers = this.directoryWatchers;
const fileWatchers = this.fileWatchers;
const directoryWatchers = this.directoryWatchers;
const ignored = this.watcherOptions.ignored;
const filter = ignored
? path => !ignored.test(path.replace(/\\/g, "/"))
: () => true;
const addToMap = (map, key, item) => {
const list = map.get(key);
if (list === undefined) {
map.set(key, [item]);
} else {
map.set(key, item);
} else if (Array.isArray(list)) {
list.push(item);
} else {
map.set(key, [list, item]);
}
};
const fileWatchersNeeded = new Map();
Expand Down Expand Up @@ -170,82 +262,47 @@ class Watchpack extends EventEmitter {
}
}
}
const newFileWatchers = new Map();
const newDirectoryWatchers = new Map();
const setupFileWatcher = (watcher, key, files) => {
watcher.on("initial-missing", type => {
for (const file of files) {
if (!missingFiles.has(file)) this._onRemove(file, file, type);
}
});
watcher.on("change", (mtime, type) => {
for (const file of files) {
this._onChange(file, mtime, file, type);
}
});
watcher.on("remove", type => {
for (const file of files) {
this._onRemove(file, file, type);
}
});
newFileWatchers.set(key, watcher);
};
const setupDirectoryWatcher = (watcher, key, directories) => {
watcher.on("initial-missing", type => {
for (const item of directories) {
this._onRemove(item, item, type);
}
});
watcher.on("change", (file, mtime, type) => {
for (const item of directories) {
this._onChange(item, mtime, file, type);
}
});
watcher.on("remove", type => {
for (const item of directories) {
this._onRemove(item, item, type);
}
});
newDirectoryWatchers.set(key, watcher);
};
// Close unneeded old watchers
const fileWatchersToClose = [];
const directoryWatchersToClose = [];
for (const [key, w] of oldFileWatchers) {
if (!fileWatchersNeeded.has(key)) {
// and update existing watchers
for (const [key, w] of fileWatchers) {
const needed = fileWatchersNeeded.get(key);
if (needed === undefined) {
w.close();
fileWatchers.delete(key);
} else {
fileWatchersToClose.push(w);
w.update(needed);
fileWatchersNeeded.delete(key);
}
}
for (const [key, w] of oldDirectoryWatchers) {
if (!directoryWatchersNeeded.has(key)) {
for (const [key, w] of directoryWatchers) {
const needed = directoryWatchersNeeded.get(key);
if (needed === undefined) {
w.close();
directoryWatchers.delete(key);
} else {
directoryWatchersToClose.push(w);
w.update(needed);
directoryWatchersNeeded.delete(key);
}
}
// Create new watchers and install handlers on these watchers
watchEventSource.batch(() => {
for (const [key, files] of fileWatchersNeeded) {
const watcher = this.watcherManager.watchFile(key, startTime);
if (watcher) {
setupFileWatcher(watcher, key, files);
fileWatchers.set(key, new WatchpackFileWatcher(this, watcher, files));
}
}
for (const [key, directories] of directoryWatchersNeeded) {
const watcher = this.watcherManager.watchDirectory(key, startTime);
if (watcher) {
setupDirectoryWatcher(watcher, key, directories);
directoryWatchers.set(
key,
new WatchpackDirectoryWatcher(this, watcher, directories)
);
}
}
});
// Close old watchers
for (const w of fileWatchersToClose) w.close();
for (const w of directoryWatchersToClose) w.close();
// Store watchers
this.fileWatchers = newFileWatchers;
this.directoryWatchers = newDirectoryWatchers;
this._missing = missingFiles;
this.startTime = startTime;
}

Expand All @@ -265,8 +322,11 @@ class Watchpack extends EventEmitter {

getTimes() {
const directoryWatchers = new Set();
addWatchersToSet(this.fileWatchers.values(), directoryWatchers);
addWatchersToSet(this.directoryWatchers.values(), directoryWatchers);
addWatchpackWatchersToSet(this.fileWatchers.values(), directoryWatchers);
addWatchpackWatchersToSet(
this.directoryWatchers.values(),
directoryWatchers
);
const obj = Object.create(null);
for (const w of directoryWatchers) {
const times = w.getTimes();
Expand All @@ -281,8 +341,11 @@ class Watchpack extends EventEmitter {
.EXISTANCE_ONLY_TIME_ENTRY;
}
const directoryWatchers = new Set();
addWatchersToSet(this.fileWatchers.values(), directoryWatchers);
addWatchersToSet(this.directoryWatchers.values(), directoryWatchers);
addWatchpackWatchersToSet(this.fileWatchers.values(), directoryWatchers);
addWatchpackWatchersToSet(
this.directoryWatchers.values(),
directoryWatchers
);
const map = new Map();
for (const w of directoryWatchers) {
const times = w.getTimeInfoEntries();
Expand Down
6 changes: 6 additions & 0 deletions test/ManyWatchers.js
View file
Open in desktop
Expand Up @@ -66,6 +66,12 @@ describe("ManyWatchers", function() {
}
w.watch({ files });
console.timeEnd("creating/closing watchers");
console.time("calling watch with the same files");
for (let i = 0; i < 2000; i++) {
w.watch({ files });
}
console.timeEnd("calling watch with the same files");

testHelper.tick(10000, () => {
console.time("detecting change event");
testHelper.file("4096/900/file");
Expand Down