Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize chunk graph algorithm #8242

Merged
merged 1 commit into from Oct 20, 2018
Merged

Optimize chunk graph algorithm #8242

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
105 changes: 62 additions & 43 deletions lib/Compilation.js
View file
Open in desktop
Expand Up @@ -88,13 +88,6 @@ const compareLocations = require("./compareLocations");
* @property {Dependency[]} dependencies
*/

/**
* @typedef {Object} AvailableModulesChunkGroupMapping
* @property {ChunkGroup} chunkGroup
* @property {Set<Module>} availableModules
* @property {boolean} needCopy
*/

/**
* @typedef {Object} DependenciesBlockLike
* @property {Dependency[]} dependencies
Expand Down Expand Up @@ -158,6 +151,16 @@ const byNameOrHash = (a, b) => {
return 0;
};

/**
* @template T
* @param {Set<T>} a first set
* @param {Set<T>} b second set
* @returns {number} cmp
*/
const bySetSize = (a, b) => {
return a.size - b.size;
};

/**
* @param {DependenciesBlockVariable[]} variables DepBlock Variables to iterate over
* @param {DepBlockVarDependenciesCallback} fn callback to apply on iterated elements
Expand Down Expand Up @@ -1779,17 +1782,26 @@ class Compilation extends Tapable {

// PART TWO
/** @type {Set<Module>} */
let availableModules;
/** @type {Set<Module>} */
let newAvailableModules;
/** @type {Queue<AvailableModulesChunkGroupMapping>} */
const queue2 = new Queue(
inputChunkGroups.map(chunkGroup => ({
chunkGroup,
availableModules: new Set(),
needCopy: true
}))
);

/**
* @typedef {Object} ChunkGroupInfo
* @property {Set<Module>} minAvailableModules current minimal set of modules available at this point
* @property {Set<Module>[]} availableModulesToBeMerged enqueued updates to the minimal set of available modules
*/

/** @type {Map<ChunkGroup, ChunkGroupInfo>} */
const chunkGroupInfoMap = new Map();

/** @type {Queue<ChunkGroup>} */
const queue2 = new Queue(inputChunkGroups);

for (const chunkGroup of inputChunkGroups) {
chunkGroupInfoMap.set(chunkGroup, {
minAvailableModules: undefined,
availableModulesToBeMerged: [new Set()]
ooflorent marked this conversation as resolved.
Show resolved Hide resolved
});
}

/**
* Helper function to check if all modules of a chunk are available
Expand Down Expand Up @@ -1824,44 +1836,43 @@ class Compilation extends Tapable {
return true;
};

/** @type {Map<ChunkGroup, Set<Module>>} */
const minAvailableModulesMap = new Map();

// Iterative traversing of the basic chunk graph
while (queue2.length) {
const queueItem = queue2.dequeue();
chunkGroup = queueItem.chunkGroup;
availableModules = queueItem.availableModules;
chunkGroup = queue2.dequeue();
const info = chunkGroupInfoMap.get(chunkGroup);
const availableModulesToBeMerged = info.availableModulesToBeMerged;
let minAvailableModules = info.minAvailableModules;

// 1. Get minimal available modules
// It doesn't make sense to traverse a chunk again with more available modules.
// This step calculates the minimal available modules and skips traversal when
// the list didn't shrink.
let minAvailableModules = minAvailableModulesMap.get(chunkGroup);
if (minAvailableModules === undefined) {
minAvailableModulesMap.set(
chunkGroup,
queueItem.needCopy ? new Set(availableModules) : availableModules
);
} else {
let deletedModules = false;
for (const m of minAvailableModules) {
if (!availableModules.has(m)) {
minAvailableModules.delete(m);
deletedModules = true;
availableModulesToBeMerged.sort(bySetSize);
let changed = false;
for (const availableModules of availableModulesToBeMerged) {
if (minAvailableModules === undefined) {
minAvailableModules = new Set(availableModules);
info.minAvailableModules = minAvailableModules;
changed = true;
} else {
for (const m of minAvailableModules) {
if (!availableModules.has(m)) {
minAvailableModules.delete(m);
changed = true;
}
}
}
if (!deletedModules) continue;
availableModules = minAvailableModules;
}
availableModulesToBeMerged.length = 0;
if (!changed) continue;

// 2. Get the edges at this point of the graph
const deps = chunkDependencies.get(chunkGroup);
if (!deps) continue;
if (deps.length === 0) continue;

// 3. Create a new Set of available modules at this points
newAvailableModules = new Set(availableModules);
newAvailableModules = new Set(minAvailableModules);
for (const chunk of chunkGroup.chunks) {
for (const m of chunk.modulesIterable) {
newAvailableModules.add(m);
Expand Down Expand Up @@ -1894,11 +1905,19 @@ class Compilation extends Tapable {

// 7. Enqueue further traversal
for (const nextChunkGroup of nextChunkGroups) {
queue2.enqueue({
chunkGroup: nextChunkGroup,
availableModules: newAvailableModules,
needCopy: nextChunkGroup.size !== 1
});
let nextInfo = chunkGroupInfoMap.get(nextChunkGroup);
if (nextInfo === undefined) {
nextInfo = {
minAvailableModules: undefined,
availableModulesToBeMerged: []
};
chunkGroupInfoMap.set(nextChunkGroup, nextInfo);
}
nextInfo.availableModulesToBeMerged.push(newAvailableModules);

// As queue deduplicates enqueued items this makes sure that a ChunkGroup
// is not enqueued twice
queue2.enqueue(nextChunkGroup);
}
}

Expand Down