Skip to content

Commit

Permalink
Merge pull request #13097 from webpack/pref/memory
Browse files Browse the repository at this point in the history 
memory and performance optimizations
  • Loading branch information
@sokra
sokra committed Apr 8, 2021
2 parents b3e26cb + cef2a8f commit abf0b4e
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 43 deletions.
16 changes: 16 additions & 0 deletions lib/CodeGenerationResults.js
View file
Expand Up @@ -27,6 +27,7 @@ class CodeGenerationResults {
* @returns {CodeGenerationResult} the CodeGenerationResult
*/
get(module, runtime) {
this._verifyAlive();
const entry = this.map.get(module);
if (entry === undefined) {
throw new Error(
Expand Down Expand Up @@ -72,6 +73,7 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
* @returns {boolean} true, when we have data for this
*/
has(module, runtime) {
this._verifyAlive();
const entry = this.map.get(module);
if (entry === undefined) {
return false;
Expand Down Expand Up @@ -142,9 +144,23 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
* @returns {void}
*/
add(module, runtime, result) {
this._verifyAlive();
const map = provide(this.map, module, () => new RuntimeSpecMap());
map.set(runtime, result);
}

_verifyAlive() {
if (this.map === undefined) {
throw new Error(
"CodeGenerationResults has been accessed after its lifetime. They are no longer available after sealing the compilation."
);
}
}

dispose() {
this.map.clear();
this.map = undefined;
}
}

module.exports = CodeGenerationResults;
20 changes: 13 additions & 7 deletions lib/Compilation.js
View file
Expand Up @@ -358,12 +358,13 @@ class Compilation {
const { fn, additionalAssets, ...remainingTap } = tap;
const additionalAssetsFn =
additionalAssets === true ? fn : additionalAssets;
let processedAssets = undefined;
const processedAssets = additionalAssetsFn ? new WeakSet() : undefined;
switch (type) {
case "sync":
if (additionalAssetsFn) {
this.hooks.processAdditionalAssets.tap(name, assets => {
if (processedAssets === this.assets) additionalAssetsFn(assets);
if (processedAssets.has(this.assets))
additionalAssetsFn(assets);
});
}
return {
Expand All @@ -375,7 +376,8 @@ class Compilation {
} catch (e) {
return callback(e);
}
processedAssets = this.assets;
if (processedAssets !== undefined)
processedAssets.add(this.assets);
const newAssets = popNewAssets(assets);
if (newAssets !== undefined) {
this.hooks.processAdditionalAssets.callAsync(
Expand All @@ -392,7 +394,7 @@ class Compilation {
this.hooks.processAdditionalAssets.tapAsync(
name,
(assets, callback) => {
if (processedAssets === this.assets)
if (processedAssets.has(this.assets))
return additionalAssetsFn(assets, callback);
callback();
}
Expand All @@ -403,7 +405,8 @@ class Compilation {
fn: (assets, callback) => {
fn(assets, err => {
if (err) return callback(err);
processedAssets = this.assets;
if (processedAssets !== undefined)
processedAssets.add(this.assets);
const newAssets = popNewAssets(assets);
if (newAssets !== undefined) {
this.hooks.processAdditionalAssets.callAsync(
Expand All @@ -419,7 +422,7 @@ class Compilation {
case "promise":
if (additionalAssetsFn) {
this.hooks.processAdditionalAssets.tapPromise(name, assets => {
if (processedAssets === this.assets)
if (processedAssets.has(this.assets))
return additionalAssetsFn(assets);
return Promise.resolve();
});
Expand All @@ -430,7 +433,8 @@ class Compilation {
const p = fn(assets);
if (!p || !p.then) return p;
return p.then(() => {
processedAssets = this.assets;
if (processedAssets !== undefined)
processedAssets.add(this.assets);
const newAssets = popNewAssets(assets);
if (newAssets !== undefined) {
return this.hooks.processAdditionalAssets.promise(
Expand Down Expand Up @@ -2128,6 +2132,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
this.assets = {};
this.assetsInfo.clear();
this.moduleGraph.removeAllModuleAttributes();
this.codeGenerationResults = undefined;
}

/**
Expand Down Expand Up @@ -2414,6 +2419,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
this.unseal();
return this.seal(callback);
}
this.codeGenerationResults.dispose();
return this.hooks.afterSeal.callAsync(err => {
if (err) {
return callback(
Expand Down
7 changes: 5 additions & 2 deletions lib/Compiler.js
View file
Expand Up @@ -359,12 +359,13 @@ class Compiler {
// e.g. move compilation specific info from Modules into ModuleGraph
_cleanupLastCompilation() {
if (this._lastCompilation !== undefined) {
for (const module of this._lastCompilation.modules) {
const c = this._lastCompilation;
for (const module of c.modules) {
ChunkGraph.clearChunkGraphForModule(module);
ModuleGraph.clearModuleGraphForModule(module);
module.cleanupForCache();
}
for (const chunk of this._lastCompilation.chunks) {
for (const chunk of c.chunks) {
ChunkGraph.clearChunkGraphForChunk(chunk);
}
this._lastCompilation = undefined;
Expand Down Expand Up @@ -820,6 +821,8 @@ ${other}`);
}
},
err => {
// Clear map to free up memory
caseInsensitiveMap.clear();
if (err) return callback(err);

this.hooks.afterEmit.callAsync(compilation, err => {
Expand Down
51 changes: 17 additions & 34 deletions lib/serialization/BinaryMiddleware.js
View file
Expand Up @@ -114,31 +114,6 @@ const identifyNumber = n => {
* @extends {SerializerMiddleware<DeserializedType, SerializedType>}
*/
class BinaryMiddleware extends SerializerMiddleware {
static optimizeSerializedData(data) {
const result = [];
const temp = [];
const flush = () => {
if (temp.length > 0) {
if (temp.length === 1) {
result.push(temp[0]);
} else {
result.push(Buffer.concat(temp));
}
temp.length = 0;
}
};
for (const item of data) {
if (Buffer.isBuffer(item)) {
temp.push(item);
} else {
flush();
result.push(item);
}
}
flush();
return result;
}

/**
* @param {DeserializedType} data data
* @param {Object} context context object
Expand All @@ -162,7 +137,7 @@ class BinaryMiddleware extends SerializerMiddleware {
/** @type {BufferSerializableType[]} */
const buffers = [];
let buffersTotalLength = 0;
const allocate = (bytesNeeded, exact = false) => {
const allocate = bytesNeeded => {
if (currentBuffer !== null) {
if (currentBuffer.length - currentPosition >= bytesNeeded) return;
flush();
Expand All @@ -171,8 +146,8 @@ class BinaryMiddleware extends SerializerMiddleware {
currentBuffer = leftOverBuffer;
leftOverBuffer = null;
} else {
currentBuffer = Buffer.allocUnsafe(
exact ? bytesNeeded : Math.max(bytesNeeded, buffersTotalLength, 1024)
currentBuffer = Buffer.allocUnsafeSlow(
Math.max(bytesNeeded, buffersTotalLength, 16384)
);
}
};
Expand Down Expand Up @@ -264,8 +239,8 @@ class BinaryMiddleware extends SerializerMiddleware {
for (const l of lengths) {
writeU32(l);
}
flush();
for (const item of serializedData) {
flush();
buffers.push(item);
}
}
Expand Down Expand Up @@ -463,11 +438,19 @@ class BinaryMiddleware extends SerializerMiddleware {
writeU32(n - 260);
}
} else if (Buffer.isBuffer(thing)) {
allocate(HEADER_SIZE + I32_SIZE, true);
writeU8(BUFFER_HEADER);
writeU32(thing.length);
flush();
buffers.push(thing);
if (thing.length < 8192) {
allocate(HEADER_SIZE + I32_SIZE + thing.length);
writeU8(BUFFER_HEADER);
writeU32(thing.length);
thing.copy(currentBuffer, currentPosition);
currentPosition += thing.length;
} else {
allocate(HEADER_SIZE + I32_SIZE);
writeU8(BUFFER_HEADER);
writeU32(thing.length);
flush();
buffers.push(thing);
}
}
break;
}
Expand Down
1 change: 1 addition & 0 deletions types.d.ts
View file
Expand Up @@ -1174,6 +1174,7 @@ declare abstract class CodeGenerationResults {
getData(module: Module, runtime: RuntimeSpec, key: string): any;
getHash(module: Module, runtime: RuntimeSpec): any;
add(module: Module, runtime: RuntimeSpec, result: CodeGenerationResult): void;
dispose(): void;
}
type CodeValue =
| undefined
Expand Down

0 comments on commit abf0b4e

Please sign in to comment.