Merge pull request #12797 from webpack/bugfix/duplicate-assets-hash

lib/optimize/RealContentHashPlugin.js

@@ -28,6 +28,27 @@ const addToList = (itemOrItems, list) => {
 	}
 };
 
+/**
+ * @template T
+ * @param {T[]} input list
+ * @param {function(T): Buffer} fn map function
+ * @returns {Buffer[]} buffers without duplicates
+ */
+const mapAndDeduplicateBuffers = (input, fn) => {
+	// Buffer.equals compares size first so this should be efficient enough
+	// If it becomes a performance problem we can use a map and group by size
+	// instead of looping over all assets.
+	const result = [];
+	outer: for (const value of input) {
+		const buf = fn(value);
+		for (const other of result) {
+			if (buf.equals(other)) continue outer;
+		}
+		result.push(buf);
+	}
+	return result;
+};
+
 /**
  * Escapes regular expression metacharacters
  * @param {string} str String to quote
@@ -331,7 +352,7 @@ ${referencingAssets
 									: computeNewContent(asset)
 							)
 						);
-						const assetsContent = assets.map(asset => {
+						const assetsContent = mapAndDeduplicateBuffers(assets, asset => {
 							if (asset.ownHashes.has(oldHash)) {
 								return asset.newSourceWithoutOwn
 									? asset.newSourceWithoutOwn.buffer()

test/__snapshots__/StatsTestCases.test.js.snap

@@ -2440,85 +2440,119 @@ LOG from webpack.FileSystemInfo
 
 exports[`StatsTestCases should print correct stats for real-content-hash 1`] = `
 "a-normal:
-  assets by path *.js 3 KiB
-    asset 18c2d7f22a461e1f021f-18c2d7.js 2.61 KiB [emitted] [immutable] [minimized] (name: runtime~main)
-    asset 3cc8faad16137711c07e-3cc8fa.js 210 bytes [emitted] [immutable] [minimized] (name: main)
-    asset b6f77787a670e97d47b5-b6f777.js 193 bytes [emitted] [immutable] [minimized] (name: lazy)
-  asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
-  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: main)
-  Entrypoint main 2.81 KiB (5.89 KiB) = 18c2d7f22a461e1f021f-18c2d7.js 2.61 KiB 3cc8faad16137711c07e-3cc8fa.js 210 bytes 1 auxiliary asset
-  runtime modules 7.37 KiB 8 modules
+  assets by path *.js 3.16 KiB
+    asset 1a429866b1a4b47a3ece-1a4298.js 2.65 KiB [emitted] [immutable] [minimized] (name: runtime)
+    asset a639a9edc4557744bf94-a639a9.js 288 bytes [emitted] [immutable] [minimized] (name: lazy)
+    asset b32c1c3b5e5cb3818671-b32c1c.js 210 bytes [emitted] [immutable] [minimized] (name: index)
+    asset 666f2b8847021ccc7608-666f2b.js 21 bytes [emitted] [immutable] [minimized] (name: a, b)
+  assets by chunk 20.4 KiB (auxiliary name: lazy)
+    asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
+    asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
+  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
+  Entrypoint index 2.85 KiB (5.89 KiB) = 1a429866b1a4b47a3ece-1a4298.js 2.65 KiB b32c1c3b5e5cb3818671-b32c1c.js 210 bytes 1 auxiliary asset
+  Entrypoint a 21 bytes = 666f2b8847021ccc7608-666f2b.js
+  Entrypoint b 21 bytes = 666f2b8847021ccc7608-666f2b.js
+  runtime modules 7.42 KiB 8 modules
   orphan modules 23 bytes [orphan] 1 module
-  cacheable modules 340 bytes (javascript) 20.4 KiB (asset)
-    javascript modules 256 bytes
+  cacheable modules 514 bytes (javascript) 26.3 KiB (asset)
+    javascript modules 388 bytes
       ./a/index.js 150 bytes [built] [code generated]
-      ./a/lazy.js + 1 modules 106 bytes [built] [code generated]
-    asset modules 84 bytes (javascript) 20.4 KiB (asset)
+      ./a/a.js 22 bytes [built] [code generated]
+      ./a/b.js 66 bytes [built] [code generated]
+      ./a/lazy.js + 1 modules 150 bytes [built] [code generated]
+    asset modules 126 bytes (javascript) 26.3 KiB (asset)
       ./a/file.jpg 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
       ./a/file.png 42 bytes (javascript) 14.6 KiB (asset) [built] [code generated]
+      ./a/file.jpg?query 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
   a-normal (webpack x.x.x) compiled successfully in X ms
 
 b-normal:
-  assets by path *.js 3 KiB
-    asset 4408ab3a86604e4c16e2-4408ab.js 2.61 KiB [emitted] [immutable] [minimized] (name: runtime~main)
-    asset 3cc8faad16137711c07e-3cc8fa.js 210 bytes [emitted] [immutable] [minimized] (name: main)
-    asset b6f77787a670e97d47b5-b6f777.js 193 bytes [emitted] [immutable] [minimized] (name: lazy)
-  asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
-  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: main)
-  Entrypoint main 2.81 KiB (5.89 KiB) = 4408ab3a86604e4c16e2-4408ab.js 2.61 KiB 3cc8faad16137711c07e-3cc8fa.js 210 bytes 1 auxiliary asset
-  runtime modules 7.37 KiB 8 modules
+  assets by path *.js 3.16 KiB
+    asset a608d8c4ea5e597b8ec0-a608d8.js 2.65 KiB [emitted] [immutable] [minimized] (name: runtime)
+    asset a639a9edc4557744bf94-a639a9.js 288 bytes [emitted] [immutable] [minimized] (name: lazy)
+    asset b32c1c3b5e5cb3818671-b32c1c.js 210 bytes [emitted] [immutable] [minimized] (name: index)
+    asset 666f2b8847021ccc7608-666f2b.js 21 bytes [emitted] [immutable] [minimized] (name: a, b)
+  assets by chunk 20.4 KiB (auxiliary name: lazy)
+    asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
+    asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
+  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
+  Entrypoint index 2.85 KiB (5.89 KiB) = a608d8c4ea5e597b8ec0-a608d8.js 2.65 KiB b32c1c3b5e5cb3818671-b32c1c.js 210 bytes 1 auxiliary asset
+  Entrypoint a 21 bytes = 666f2b8847021ccc7608-666f2b.js
+  Entrypoint b 21 bytes = 666f2b8847021ccc7608-666f2b.js
+  runtime modules 7.42 KiB 8 modules
   orphan modules 19 bytes [orphan] 1 module
-  cacheable modules 295 bytes (javascript) 20.4 KiB (asset)
-    javascript modules 211 bytes
+  cacheable modules 469 bytes (javascript) 26.3 KiB (asset)
+    javascript modules 343 bytes
       ./b/index.js 109 bytes [built] [code generated]
-      ./b/lazy.js + 1 modules 102 bytes [built] [code generated]
-    asset modules 84 bytes (javascript) 20.4 KiB (asset)
+      ./b/a.js 22 bytes [built] [code generated]
+      ./b/b.js 66 bytes [built] [code generated]
+      ./b/lazy.js + 1 modules 146 bytes [built] [code generated]
+    asset modules 126 bytes (javascript) 26.3 KiB (asset)
       ./b/file.jpg 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
       ./b/file.png 42 bytes (javascript) 14.6 KiB (asset) [built] [code generated]
+      ./b/file.jpg?query 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
   b-normal (webpack x.x.x) compiled successfully in X ms
 
 a-source-map:
-  assets by path *.js 3.17 KiB
-    asset 9351b9819e587de6a617-9351b9.js 2.66 KiB [emitted] [immutable] [minimized] (name: runtime~main)
-      sourceMap 9351b9819e587de6a617-9351b9.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime~main)
-    asset b8bfcec62cdd15c9a840-b8bfce.js 266 bytes [emitted] [immutable] [minimized] (name: main)
-      sourceMap b8bfcec62cdd15c9a840-b8bfce.js.map 366 bytes [emitted] [dev] (auxiliary name: main)
-    asset c7c0f8bb2e61b59a89f5-c7c0f8.js 249 bytes [emitted] [immutable] [minimized] (name: lazy)
-      sourceMap c7c0f8bb2e61b59a89f5-c7c0f8.js.map 331 bytes [emitted] [dev] (auxiliary name: lazy)
-  asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
-  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: main)
-  Entrypoint main 2.92 KiB (20.7 KiB) = 9351b9819e587de6a617-9351b9.js 2.66 KiB b8bfcec62cdd15c9a840-b8bfce.js 266 bytes 3 auxiliary assets
-  runtime modules 7.37 KiB 8 modules
+  assets by path *.js 3.38 KiB
+    asset 2064caf3f763dcbec7da-2064ca.js 2.7 KiB [emitted] [immutable] [minimized] (name: runtime)
+      sourceMap 2064caf3f763dcbec7da-2064ca.js.map 14.6 KiB [emitted] [dev] (auxiliary name: runtime)
+    asset 05c637d15dff2d0dc5fd-05c637.js 344 bytes [emitted] [immutable] [minimized] (name: lazy)
+      sourceMap 05c637d15dff2d0dc5fd-05c637.js.map 399 bytes [emitted] [dev] (auxiliary name: lazy)
+    asset 793a9ff3858dbc0b791d-793a9f.js 266 bytes [emitted] [immutable] [minimized] (name: index)
+      sourceMap 793a9ff3858dbc0b791d-793a9f.js.map 366 bytes [emitted] [dev] (auxiliary name: index)
+    asset 222c2acc68675174e6b2-222c2a.js 77 bytes [emitted] [immutable] [minimized] (name: a, b)
+      sourceMap 222c2acc68675174e6b2-222c2a.js.map 254 bytes [emitted] [dev] (auxiliary name: a, b)
+  assets by chunk 20.4 KiB (auxiliary name: lazy)
+    asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
+    asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
+  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
+  Entrypoint index 2.96 KiB (20.8 KiB) = 2064caf3f763dcbec7da-2064ca.js 2.7 KiB 793a9ff3858dbc0b791d-793a9f.js 266 bytes 3 auxiliary assets
+  Entrypoint a 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
+  Entrypoint b 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
+  runtime modules 7.42 KiB 8 modules
   orphan modules 23 bytes [orphan] 1 module
-  cacheable modules 340 bytes (javascript) 20.4 KiB (asset)
-    javascript modules 256 bytes
+  cacheable modules 514 bytes (javascript) 26.3 KiB (asset)
+    javascript modules 388 bytes
       ./a/index.js 150 bytes [built] [code generated]
-      ./a/lazy.js + 1 modules 106 bytes [built] [code generated]
-    asset modules 84 bytes (javascript) 20.4 KiB (asset)
+      ./a/a.js 22 bytes [built] [code generated]
+      ./a/b.js 66 bytes [built] [code generated]
+      ./a/lazy.js + 1 modules 150 bytes [built] [code generated]
+    asset modules 126 bytes (javascript) 26.3 KiB (asset)
       ./a/file.jpg 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
       ./a/file.png 42 bytes (javascript) 14.6 KiB (asset) [built] [code generated]
+      ./a/file.jpg?query 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
   a-source-map (webpack x.x.x) compiled successfully in X ms
 
 b-source-map:
-  assets by path *.js 3.17 KiB
-    asset ff44eb1f030374616671-ff44eb.js 2.66 KiB [emitted] [immutable] [minimized] (name: runtime~main)
-      sourceMap ff44eb1f030374616671-ff44eb.js.map 14.5 KiB [emitted] [dev] (auxiliary name: runtime~main)
-    asset b8bfcec62cdd15c9a840-b8bfce.js 266 bytes [emitted] [immutable] [minimized] (name: main)
-      sourceMap b8bfcec62cdd15c9a840-b8bfce.js.map 323 bytes [emitted] [dev] (auxiliary name: main)
-    asset c7c0f8bb2e61b59a89f5-c7c0f8.js 249 bytes [emitted] [immutable] [minimized] (name: lazy)
-      sourceMap c7c0f8bb2e61b59a89f5-c7c0f8.js.map 327 bytes [emitted] [dev] (auxiliary name: lazy)
-  asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
-  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: main)
-  Entrypoint main 2.92 KiB (20.7 KiB) = ff44eb1f030374616671-ff44eb.js 2.66 KiB b8bfcec62cdd15c9a840-b8bfce.js 266 bytes 3 auxiliary assets
-  runtime modules 7.37 KiB 8 modules
+  assets by path *.js 3.38 KiB
+    asset 5a04964d0347bb4c6f39-5a0496.js 2.7 KiB [emitted] [immutable] [minimized] (name: runtime)
+      sourceMap 5a04964d0347bb4c6f39-5a0496.js.map 14.6 KiB [emitted] [dev] (auxiliary name: runtime)
+    asset 05c637d15dff2d0dc5fd-05c637.js 344 bytes [emitted] [immutable] [minimized] (name: lazy)
+      sourceMap 05c637d15dff2d0dc5fd-05c637.js.map 395 bytes [emitted] [dev] (auxiliary name: lazy)
+    asset 793a9ff3858dbc0b791d-793a9f.js 266 bytes [emitted] [immutable] [minimized] (name: index)
+      sourceMap 793a9ff3858dbc0b791d-793a9f.js.map 323 bytes [emitted] [dev] (auxiliary name: index)
+    asset 222c2acc68675174e6b2-222c2a.js 77 bytes [emitted] [immutable] [minimized] (name: a, b)
+      sourceMap 222c2acc68675174e6b2-222c2a.js.map 254 bytes [emitted] [dev] (auxiliary name: a, b)
+  assets by chunk 20.4 KiB (auxiliary name: lazy)
+    asset 89a353e9c515885abd8e.png 14.6 KiB [emitted] [immutable] [from: file.png] (auxiliary name: lazy)
+    asset 7382fad5b015914e0811.jpg?query 5.89 KiB [cached] [immutable] [from: file.jpg?query] (auxiliary name: lazy)
+  asset 7382fad5b015914e0811.jpg 5.89 KiB [emitted] [immutable] [from: file.jpg] (auxiliary name: index)
+  Entrypoint index 2.96 KiB (20.8 KiB) = 5a04964d0347bb4c6f39-5a0496.js 2.7 KiB 793a9ff3858dbc0b791d-793a9f.js 266 bytes 3 auxiliary assets
+  Entrypoint a 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
+  Entrypoint b 77 bytes (254 bytes) = 222c2acc68675174e6b2-222c2a.js 1 auxiliary asset
+  runtime modules 7.42 KiB 8 modules
   orphan modules 19 bytes [orphan] 1 module
-  cacheable modules 295 bytes (javascript) 20.4 KiB (asset)
-    javascript modules 211 bytes
+  cacheable modules 469 bytes (javascript) 26.3 KiB (asset)
+    javascript modules 343 bytes
       ./b/index.js 109 bytes [built] [code generated]
-      ./b/lazy.js + 1 modules 102 bytes [built] [code generated]
-    asset modules 84 bytes (javascript) 20.4 KiB (asset)
+      ./b/a.js 22 bytes [built] [code generated]
+      ./b/b.js 66 bytes [built] [code generated]
+      ./b/lazy.js + 1 modules 146 bytes [built] [code generated]
+    asset modules 126 bytes (javascript) 26.3 KiB (asset)
       ./b/file.jpg 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
       ./b/file.png 42 bytes (javascript) 14.6 KiB (asset) [built] [code generated]
+      ./b/file.jpg?query 42 bytes (javascript) 5.89 KiB (asset) [built] [code generated]
   b-source-map (webpack x.x.x) compiled successfully in X ms"
 `;
 

test/statsCases/real-content-hash/a/a.js

@@ -0,0 +1 @@
+console.log("hello");

test/statsCases/real-content-hash/a/b.js

@@ -0,0 +1,4 @@
+// yes that's important
+if (true) {
+	console.log("hel" + "lo");
+}

test/statsCases/real-content-hash/a/lazy.js

@@ -1,3 +1,3 @@
 import test from "./module";
 import url from "./file.png";
-console.log(test, url);
+console.log(test, url, new URL("file.jpg?query", import.meta.url));

test/statsCases/real-content-hash/b/a.js

@@ -0,0 +1 @@
+console.log("hello");

test/statsCases/real-content-hash/b/b.js

@@ -0,0 +1,4 @@
+// yes that's important
+if (true) {
+	console.log("hel" + "lo");
+}

test/statsCases/real-content-hash/b/lazy.js

@@ -1,3 +1,3 @@
 import test from "./module";
 import url from "./file.png";
-console.log(test, url);
+console.log(test, url, new URL("file.jpg?query", import.meta.url));

test/statsCases/real-content-hash/test.config.js

@@ -1,3 +1,12 @@
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const hashedFiles = {
+	"file.jpg": a => a.name.endsWith(".jpg"),
+	"file.png": a => a.name.endsWith(".png")
+};
+
 module.exports = {
 	validate(stats) {
 		for (let i = 0; i < 4; i += 2) {
@@ -7,9 +16,28 @@ module.exports = {
 			const b = stats.stats[i + 1].toJson({
 				assets: true
 			});
-			expect(Object.keys(a.assetsByChunkName).length).toBe(3);
+			expect(Object.keys(a.assetsByChunkName).length).toBe(5);
 			expect(a.assetsByChunkName.main).toEqual(b.assetsByChunkName.main);
 			expect(a.assetsByChunkName.lazy).toEqual(b.assetsByChunkName.lazy);
+			expect(a.assetsByChunkName.a).toEqual(b.assetsByChunkName.a);
+			expect(a.assetsByChunkName.b).toEqual(b.assetsByChunkName.b);
+			expect(a.assetsByChunkName.a).toEqual(a.assetsByChunkName.b);
+		}
+		for (let i = 0; i < 4; i++) {
+			const statsData = stats.stats[i].toJson({
+				assets: true
+			});
+			for (const name of Object.keys(hashedFiles)) {
+				const asset = statsData.assets.find(hashedFiles[name]);
+				expect(asset).not.toBe(undefined);
+				const content = fs.readFileSync(path.resolve(__dirname, "a", name));
+				const hash = crypto
+					.createHash("md4")
+					.update(content)
+					.digest("hex")
+					.slice(0, 20);
+				expect(asset.name.slice(0, 20)).toBe(hash);
+			}
 		}
 	}
 };

test/statsCases/real-content-hash/webpack.config.js

@@ -1,9 +1,22 @@
 const path = require("path");
 
+/** @type {import("../../../").Configuration} */
 const base = {
 	mode: "production",
-	entry: "./index",
+	entry: {
+		index: {
+			import: "./index",
+			runtime: "runtime"
+		},
+		a: "./a",
+		b: "./b"
+	},
 	module: {
+		generator: {
+			asset: {
+				filename: "[hash][ext][query]"
+			}
+		},
 		rules: [
 			{
 				test: /\.(png|jpg)$/,
@@ -12,11 +25,11 @@ const base = {
 		]
 	},
 	optimization: {
-		runtimeChunk: true,
 		minimize: true
 	},
 	stats: {
-		relatedAssets: true
+		relatedAssets: true,
+		cachedAssets: true
 	}
 };