feat: deduplicate assets with buffer source

docs/05-plugin-development.md

@@ -732,7 +732,7 @@ If there are no dynamic imports, this will create exactly three chunks where the
 
 Note that even though any module id can be used in `implicitlyLoadedAfterOneOf`, Rollup will throw an error if such an id cannot be uniquely associated with a chunk, e.g. because the `id` cannot be reached implicitly or explicitly from the existing static entry points, or because the file is completely tree-shaken. Using only entry points, either defined by the user or of previously emitted chunks, will always work, though.
 
-If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same `string` source even if the `name` does not match. If the source is not a string but a typed array or `Buffer`, no deduplication will occur for performance reasons. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
+If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same source even if the `name` does not match. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
 
 #### `this.error`
 

src/utils/FileEmitter.ts

@@ -28,33 +28,34 @@ import { extname } from './path';
 import { isPathFragment } from './relativeId';
 import { makeUnique, renderNamePattern } from './renderNamePattern';
 
-function generateAssetFileName(
+function getSourceHash(source: string | Uint8Array, size?: number): string {
+	return createHash()
+		.update(source)
+		.digest('hex')
+		.slice(0, Math.max(0, size || defaultHashSize));
+}
+
+function pregenerateAssetFileName(
 	name: string | undefined,
 	source: string | Uint8Array,
-	outputOptions: NormalizedOutputOptions,
-	bundle: OutputBundleWithPlaceholders
-): string {
+	outputOptions: NormalizedOutputOptions
+): { fileName: string; sourceHash: string | undefined } {
+	let sourceHash: string | undefined;
 	const emittedName = outputOptions.sanitizeFileName(name || 'asset');
-	return makeUnique(
-		renderNamePattern(
-			typeof outputOptions.assetFileNames === 'function'
-				? outputOptions.assetFileNames({ name, source, type: 'asset' })
-				: outputOptions.assetFileNames,
-			'output.assetFileNames',
-			{
-				ext: () => extname(emittedName).slice(1),
-				extname: () => extname(emittedName),
-				hash: size =>
-					createHash()
-						.update(source)
-						.digest('hex')
-						.slice(0, Math.max(0, size || defaultHashSize)),
-				name: () =>
-					emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length))
-			}
-		),
-		bundle
+	const fileName = renderNamePattern(
+		typeof outputOptions.assetFileNames === 'function'
+			? outputOptions.assetFileNames({ name, source, type: 'asset' })
+			: outputOptions.assetFileNames,
+		'output.assetFileNames',
+		{
+			ext: () => extname(emittedName).slice(1),
+			extname: () => extname(emittedName),
+			hash: size => (sourceHash = getSourceHash(source, size)),
+			name: () =>
+				emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length))
+		}
 	);
+	return { fileName, sourceHash };
 }
 
 function reserveFileNameInBundle(
@@ -246,9 +247,6 @@ export class FileEmitter {
 		for (const emittedFile of this.filesByReferenceId.values()) {
 			if (emittedFile.fileName) {
 				reserveFileNameInBundle(emittedFile.fileName, output, this.options.onwarn);
-				if (emittedFile.type === 'asset' && typeof emittedFile.source === 'string') {
-					fileNamesBySource.set(emittedFile.source, emittedFile.fileName);
-				}
 			}
 		}
 		for (const [referenceId, consumedFile] of this.filesByReferenceId) {
@@ -332,17 +330,24 @@ export class FileEmitter {
 		referenceId: string,
 		{ bundle, fileNamesBySource, outputOptions }: FileEmitterOutput
 	): void {
-		const fileName =
-			consumedFile.fileName ||
-			(typeof source === 'string' && fileNamesBySource.get(source)) ||
-			generateAssetFileName(consumedFile.name, source, outputOptions, bundle);
+		let fileName = consumedFile.fileName;
+
+		// Deduplicate assets if an explicit fileName is not provided
+		if (!fileName) {
+			const pregenerated = pregenerateAssetFileName(consumedFile.name, source, outputOptions);
+			// Reuse source hash if it was computed as part of the asset file name
+			const sourceHash = pregenerated.sourceHash ?? getSourceHash(source);
+			fileName = fileNamesBySource.get(sourceHash);
+			if (!fileName) {
+				fileName = makeUnique(pregenerated.fileName, bundle);
+				fileNamesBySource.set(sourceHash, fileName);
+			}
+		}
 
 		// We must not modify the original assets to avoid interaction between outputs
 		const assetWithFileName = { ...consumedFile, fileName, source };
 		this.filesByReferenceId.set(referenceId, assetWithFileName);
-		if (typeof source === 'string') {
-			fileNamesBySource.set(source, fileName);
-		}
+
 		bundle[fileName] = {
 			fileName,
 			name: consumedFile.name,

test/cli/samples/watch/bundle-error/main.js

@@ -0,0 +1 @@
+export default 42;

test/function/samples/emit-file/deduplicate-binary-asset/_config.js

@@ -0,0 +1,18 @@
+const source = Buffer.from('abc');
+
+module.exports = {
+	description: 'warns if multiple files with the same name are emitted',
+	options: {
+		input: 'main.js',
+		plugins: [
+			{
+				buildStart() {
+					this.emitFile({ type: 'asset', name: 'buildStart', source });
+				},
+				generateBundle() {
+					this.emitFile({ type: 'asset', name: 'generateBundle', source });
+				}
+			}
+		]
+	}
+};

test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js

@@ -0,0 +1,9 @@
+define(['exports'], function (exports) { 'use strict';
+
+	function hi() { return 2 }
+
+	exports.hi = hi;
+
+	Object.defineProperty(exports, '__esModule', { value: true });
+
+});

test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile

@@ -0,0 +1 @@
+abc

test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js

@@ -0,0 +1,7 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+function hi() { return 2 }
+
+exports.hi = hi;

test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile

@@ -0,0 +1 @@
+abc

test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js

@@ -0,0 +1,3 @@
+function hi() { return 2 }
+
+export { hi };

test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile

@@ -0,0 +1 @@
+abc

test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js

@@ -0,0 +1,12 @@
+System.register([], function (exports) {
+	'use strict';
+	return {
+		execute: function () {
+
+			exports('hi', hi);
+
+			function hi() { return 2 }
+
+		}
+	};
+});

test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile

@@ -0,0 +1 @@
+abc

test/function/samples/emit-file/deduplicate-binary-asset/main.js

@@ -0,0 +1 @@
+export function hi() { return 2 }