feat: deduplicate assets with buffer source (#4712)

* feat: deduplicate binary assets option

* docs: deduplicateBinaryAssets option

* test: fix previous tests after new option

* test: sample using deduplicateBinaryAssets

* chore: remove option and simplify code

* docs: update emitFile deduplication note

* chore: remove missplaced test case

* test: extend chunking-form deduplicate-assets

* chore: simplify sourceHash generation

* refactor: precompute hash then slice

Co-authored-by: Lukas Taegert-Atkinson <lukastaegert@users.noreply.github.com>

docs/05-plugin-development.md

@@ -732,7 +732,7 @@ If there are no dynamic imports, this will create exactly three chunks where the
 
 Note that even though any module id can be used in `implicitlyLoadedAfterOneOf`, Rollup will throw an error if such an id cannot be uniquely associated with a chunk, e.g. because the `id` cannot be reached implicitly or explicitly from the existing static entry points, or because the file is completely tree-shaken. Using only entry points, either defined by the user or of previously emitted chunks, will always work, though.
 
-If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same `string` source even if the `name` does not match. If the source is not a string but a typed array or `Buffer`, no deduplication will occur for performance reasons. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
+If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same source even if the `name` does not match. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
 
 #### `this.error`
 

src/utils/FileEmitter.ts

@@ -28,9 +28,14 @@ import { extname } from './path';
 import { isPathFragment } from './relativeId';
 import { makeUnique, renderNamePattern } from './renderNamePattern';
 
+function getSourceHash(source: string | Uint8Array): string {
+	return createHash().update(source).digest('hex');
+}
+
 function generateAssetFileName(
 	name: string | undefined,
 	source: string | Uint8Array,
+	sourceHash: string,
 	outputOptions: NormalizedOutputOptions,
 	bundle: OutputBundleWithPlaceholders
 ): string {
@@ -44,11 +49,7 @@ function generateAssetFileName(
 			{
 				ext: () => extname(emittedName).slice(1),
 				extname: () => extname(emittedName),
-				hash: size =>
-					createHash()
-						.update(source)
-						.digest('hex')
-						.slice(0, Math.max(0, size || defaultHashSize)),
+				hash: size => sourceHash.slice(0, Math.max(0, size || defaultHashSize)),
 				name: () =>
 					emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length))
 			}
@@ -246,9 +247,6 @@ export class FileEmitter {
 		for (const emittedFile of this.filesByReferenceId.values()) {
 			if (emittedFile.fileName) {
 				reserveFileNameInBundle(emittedFile.fileName, output, this.options.onwarn);
-				if (emittedFile.type === 'asset' && typeof emittedFile.source === 'string') {
-					fileNamesBySource.set(emittedFile.source, emittedFile.fileName);
-				}
 			}
 		}
 		for (const [referenceId, consumedFile] of this.filesByReferenceId) {
@@ -332,17 +330,28 @@ export class FileEmitter {
 		referenceId: string,
 		{ bundle, fileNamesBySource, outputOptions }: FileEmitterOutput
 	): void {
-		const fileName =
-			consumedFile.fileName ||
-			(typeof source === 'string' && fileNamesBySource.get(source)) ||
-			generateAssetFileName(consumedFile.name, source, outputOptions, bundle);
+		let fileName = consumedFile.fileName;
+
+		// Deduplicate assets if an explicit fileName is not provided
+		if (!fileName) {
+			const sourceHash = getSourceHash(source);
+			fileName = fileNamesBySource.get(sourceHash);
+			if (!fileName) {
+				fileName = generateAssetFileName(
+					consumedFile.name,
+					source,
+					sourceHash,
+					outputOptions,
+					bundle
+				);
+				fileNamesBySource.set(sourceHash, fileName);
+			}
+		}
 
 		// We must not modify the original assets to avoid interaction between outputs
 		const assetWithFileName = { ...consumedFile, fileName, source };
 		this.filesByReferenceId.set(referenceId, assetWithFileName);
-		if (typeof source === 'string') {
-			fileNamesBySource.set(source, fileName);
-		}
+
 		bundle[fileName] = {
 			fileName,
 			name: consumedFile.name,

test/chunking-form/samples/emit-file/deduplicate-assets/_config.js

@@ -5,10 +5,45 @@ module.exports = {
 		plugins: {
 			buildStart() {
 				this.emitFile({ type: 'asset', name: 'string.txt', source: 'string' });
+				this.emitFile({ type: 'asset', name: 'stringSameSource.txt', source: 'string' });
+				this.emitFile({
+					type: 'asset',
+					name: 'sameStringAsBuffer.txt',
+					source: Buffer.from('string') // Test cross Buffer/string deduplication
+				});
+				// Different string source
 				this.emitFile({ type: 'asset', name: 'otherString.txt', source: 'otherString' });
 
+				const bufferSource = () => Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72]);
+				this.emitFile({
+					type: 'asset',
+					name: 'buffer.txt',
+					source: bufferSource()
+				});
+				this.emitFile({
+					type: 'asset',
+					name: 'bufferSameSource.txt',
+					source: bufferSource()
+				});
+				this.emitFile({
+					type: 'asset',
+					name: 'sameBufferAsString.txt',
+					source: bufferSource().toString() // Test cross Buffer/string deduplication
+				});
+				// Different buffer source
+				this.emitFile({
+					type: 'asset',
+					name: 'otherBuffer.txt',
+					source: Buffer.from('otherBuffer')
+				});
+
 				// specific file names will not be deduplicated
 				this.emitFile({ type: 'asset', fileName: 'named/string.txt', source: 'named' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'named/buffer.txt',
+					source: bufferSource()
+				});
 				return null;
 			}
 		}

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/buffer-d0ca8c2a.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherBuffer-e8d9b528.txt

@@ -0,0 +1 @@
+otherBuffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/buffer.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/buffer-d0ca8c2a.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherBuffer-e8d9b528.txt

@@ -0,0 +1 @@
+otherBuffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/buffer.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/buffer-d0ca8c2a.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherBuffer-e8d9b528.txt

@@ -0,0 +1 @@
+otherBuffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/buffer.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/buffer-d0ca8c2a.txt

@@ -0,0 +1 @@
+buffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherBuffer-e8d9b528.txt

@@ -0,0 +1 @@
+otherBuffer

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/buffer.txt

@@ -0,0 +1 @@
+buffer

test/cli/samples/watch/bundle-error/main.js

@@ -0,0 +1 @@
+export default 42;