From 8b248a74c9bf7310f051d1ba5311623c6ba2eeab Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 07:15:31 +0100 Subject: [PATCH 01/10] feat: deduplicate binary assets option --- src/rollup/types.d.ts | 2 + src/utils/FileEmitter.ts | 93 ++++++++++++++------- src/utils/options/mergeOptions.ts | 1 + src/utils/options/normalizeOutputOptions.ts | 1 + 4 files changed, 67 insertions(+), 30 deletions(-) diff --git a/src/rollup/types.d.ts b/src/rollup/types.d.ts index 70805021277..d0c73229131 100644 --- a/src/rollup/types.d.ts +++ b/src/rollup/types.d.ts @@ -632,6 +632,7 @@ export interface OutputOptions { banner?: string | AddonFunction; chunkFileNames?: string | ((chunkInfo: PreRenderedChunk) => string); compact?: boolean; + deduplicateBinaryAssets?: boolean; // only required for bundle.write dir?: string; /** @deprecated Use the "renderDynamicImport" plugin hook instead. */ @@ -685,6 +686,7 @@ export interface NormalizedOutputOptions { banner: AddonFunction; chunkFileNames: string | ((chunkInfo: PreRenderedChunk) => string); compact: boolean; + deduplicateBinaryAssets: boolean; dir: string | undefined; /** @deprecated Use the "renderDynamicImport" plugin hook instead. */ dynamicImportFunction: string | undefined; diff --git a/src/utils/FileEmitter.ts b/src/utils/FileEmitter.ts index 3880a9ea279..4fb0274bf83 100644 --- a/src/utils/FileEmitter.ts +++ b/src/utils/FileEmitter.ts @@ -28,33 +28,45 @@ import { extname } from './path'; import { isPathFragment } from './relativeId'; import { makeUnique, renderNamePattern } from './renderNamePattern'; +function getSourceHash(source: string | Uint8Array, size?: number): string { + return createHash() + .update(source) + .digest('hex') + .slice(0, Math.max(0, size || defaultHashSize)); +} + +function prepareAssetFileName( + name: string | undefined, + source: string | Uint8Array, + outputOptions: NormalizedOutputOptions +): { fileName: string; sourceHash: string | undefined } { + let sourceHash: string | undefined; + const emittedName = outputOptions.sanitizeFileName(name || 'asset'); + const fileName = renderNamePattern( + typeof outputOptions.assetFileNames === 'function' + ? outputOptions.assetFileNames({ name, source, type: 'asset' }) + : outputOptions.assetFileNames, + 'output.assetFileNames', + { + ext: () => extname(emittedName).slice(1), + extname: () => extname(emittedName), + hash: size => (sourceHash = getSourceHash(source, size)), + name: () => + emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length)) + } + ); + return { fileName, sourceHash }; +} + function generateAssetFileName( name: string | undefined, source: string | Uint8Array, outputOptions: NormalizedOutputOptions, - bundle: OutputBundleWithPlaceholders + bundle: OutputBundleWithPlaceholders, + preparedFileName?: string ): string { - const emittedName = outputOptions.sanitizeFileName(name || 'asset'); - return makeUnique( - renderNamePattern( - typeof outputOptions.assetFileNames === 'function' - ? outputOptions.assetFileNames({ name, source, type: 'asset' }) - : outputOptions.assetFileNames, - 'output.assetFileNames', - { - ext: () => extname(emittedName).slice(1), - extname: () => extname(emittedName), - hash: size => - createHash() - .update(source) - .digest('hex') - .slice(0, Math.max(0, size || defaultHashSize)), - name: () => - emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length)) - } - ), - bundle - ); + const fileName = preparedFileName || prepareAssetFileName(name, source, outputOptions).fileName; + return makeUnique(fileName, bundle); } function reserveFileNameInBundle( @@ -246,9 +258,6 @@ export class FileEmitter { for (const emittedFile of this.filesByReferenceId.values()) { if (emittedFile.fileName) { reserveFileNameInBundle(emittedFile.fileName, output, this.options.onwarn); - if (emittedFile.type === 'asset' && typeof emittedFile.source === 'string') { - fileNamesBySource.set(emittedFile.source, emittedFile.fileName); - } } } for (const [referenceId, consumedFile] of this.filesByReferenceId) { @@ -332,17 +341,41 @@ export class FileEmitter { referenceId: string, { bundle, fileNamesBySource, outputOptions }: FileEmitterOutput ): void { - const fileName = - consumedFile.fileName || - (typeof source === 'string' && fileNamesBySource.get(source)) || - generateAssetFileName(consumedFile.name, source, outputOptions, bundle); + let preparedFileName: string | undefined; + let sourceHash: string | undefined; + + let fileName = consumedFile.fileName; + + if (!fileName) { + if (outputOptions.deduplicateBinaryAssets) { + // Reuse source hash if it is going to be needed in the asset file name + const prepared = prepareAssetFileName(consumedFile.name, source, outputOptions); + preparedFileName = prepared.fileName; + sourceHash = prepared.sourceHash || getSourceHash(source); + fileName = fileNamesBySource.get(sourceHash); + } else if (typeof source === 'string') { + fileName = fileNamesBySource.get(source); + } + } + + fileName ||= generateAssetFileName( + consumedFile.name, + source, + outputOptions, + bundle, + preparedFileName + ); // We must not modify the original assets to avoid interaction between outputs const assetWithFileName = { ...consumedFile, fileName, source }; this.filesByReferenceId.set(referenceId, assetWithFileName); - if (typeof source === 'string') { + + if (sourceHash) { + fileNamesBySource.set(sourceHash, fileName); + } else if (typeof source === 'string') { fileNamesBySource.set(source, fileName); } + bundle[fileName] = { fileName, name: consumedFile.name, diff --git a/src/utils/options/mergeOptions.ts b/src/utils/options/mergeOptions.ts index be429cfb598..ccd2493d08a 100644 --- a/src/utils/options/mergeOptions.ts +++ b/src/utils/options/mergeOptions.ts @@ -227,6 +227,7 @@ async function mergeOutputOptions( banner: getOption('banner'), chunkFileNames: getOption('chunkFileNames'), compact: getOption('compact'), + deduplicateBinaryAssets: getOption('deduplicateBinaryAssets'), dir: getOption('dir'), dynamicImportFunction: getOption('dynamicImportFunction'), dynamicImportInCjs: getOption('dynamicImportInCjs'), diff --git a/src/utils/options/normalizeOutputOptions.ts b/src/utils/options/normalizeOutputOptions.ts index e03e92eba9b..66b97059744 100644 --- a/src/utils/options/normalizeOutputOptions.ts +++ b/src/utils/options/normalizeOutputOptions.ts @@ -45,6 +45,7 @@ export async function normalizeOutputOptions( banner: getAddon(config, 'banner'), chunkFileNames: config.chunkFileNames ?? '[name]-[hash].js', compact, + deduplicateBinaryAssets: config.deduplicateBinaryAssets ?? false, dir: getDir(config, file), dynamicImportFunction: getDynamicImportFunction(config, inputOptions, format), dynamicImportInCjs: config.dynamicImportInCjs ?? true, From 670cdf770bee328e169d5eafc9aaf50f4f01061c Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 07:18:02 +0100 Subject: [PATCH 02/10] docs: deduplicateBinaryAssets option --- docs/02-javascript-api.md | 1 + docs/999-big-list-of-options.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/docs/02-javascript-api.md b/docs/02-javascript-api.md index c4cc17bfae8..321d0643c17 100755 --- a/docs/02-javascript-api.md +++ b/docs/02-javascript-api.md @@ -181,6 +181,7 @@ const outputOptions = { noConflict, preferConst, sanitizeFileName, + deduplicateBinaryAssets, strict, systemNullSetters }; diff --git a/docs/999-big-list-of-options.md b/docs/999-big-list-of-options.md index 0eff6b40e3c..3c6ac6bfa20 100755 --- a/docs/999-big-list-of-options.md +++ b/docs/999-big-list-of-options.md @@ -1468,6 +1468,12 @@ export default { // -> define(['./chunk-or-local-file.js', 'dependency', 'third/dependency'],... ``` +#### output.deduplicateBinaryAssets + +Type: `boolean`
Default: `false` + +Set to `true` to also deduplicate assets when calling `emitFile` with a `Buffer` as source. By default only string sources are deduplicated. + #### output.esModule Type: `boolean | "if-default-prop"`
CLI: `--esModule`/`--no-esModule`
Default: `"if-default-prop"` From 22f9af6b75a7d2177c0a789cf07ad41c23eb2011 Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 08:25:19 +0100 Subject: [PATCH 03/10] test: fix previous tests after new option --- test/function/samples/output-options-hook/_config.js | 1 + test/misc/optionList.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/function/samples/output-options-hook/_config.js b/test/function/samples/output-options-hook/_config.js index ed8e82de12c..bbcc8cf074c 100644 --- a/test/function/samples/output-options-hook/_config.js +++ b/test/function/samples/output-options-hook/_config.js @@ -24,6 +24,7 @@ module.exports = { assetFileNames: 'assets/[name]-[hash][extname]', chunkFileNames: '[name]-[hash].js', compact: false, + deduplicateBinaryAssets: false, dynamicImportInCjs: true, entryFileNames: '[name].js', esModule: 'if-default-prop', diff --git a/test/misc/optionList.js b/test/misc/optionList.js index 9cf18ae43ca..f7770c19c85 100644 --- a/test/misc/optionList.js +++ b/test/misc/optionList.js @@ -1,6 +1,6 @@ exports.input = 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch'; exports.flags = - 'acorn, acornInjectPlugins, amd, assetFileNames, banner, bundleConfigAsCjs, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, dynamicImportInCjs, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalImportAssertions, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, generatedCode, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; + 'acorn, acornInjectPlugins, amd, assetFileNames, banner, bundleConfigAsCjs, c, cache, chunkFileNames, compact, config, configPlugin, context, d, deduplicateBinaryAssets, dir, dynamicImportFunction, dynamicImportInCjs, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalImportAssertions, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, generatedCode, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; exports.output = - 'amd, assetFileNames, banner, chunkFileNames, compact, dir, dynamicImportFunction, dynamicImportInCjs, entryFileNames, esModule, exports, extend, externalImportAssertions, externalLiveBindings, file, footer, format, freeze, generatedCode, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate'; + 'amd, assetFileNames, banner, chunkFileNames, compact, deduplicateBinaryAssets, dir, dynamicImportFunction, dynamicImportInCjs, entryFileNames, esModule, exports, extend, externalImportAssertions, externalLiveBindings, file, footer, format, freeze, generatedCode, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate'; From 131b6b25080a2e43bc7d0829f3766b7937637d27 Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 08:37:19 +0100 Subject: [PATCH 04/10] test: sample using deduplicateBinaryAssets --- .../deduplicate-binary-asset/_config.js | 18 ++++++++++++++++++ .../_expected/amd/main.js | 9 +++++++++ .../_expected/amd/myfile | 1 + .../_expected/cjs/main.js | 7 +++++++ .../_expected/cjs/myfile | 1 + .../_expected/es/main.js | 3 +++ .../_expected/es/myfile | 1 + .../_expected/system/main.js | 12 ++++++++++++ .../_expected/system/myfile | 1 + .../emit-file/deduplicate-binary-asset/main.js | 1 + 10 files changed, 54 insertions(+) create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_config.js create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile create mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/main.js diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_config.js b/test/function/samples/emit-file/deduplicate-binary-asset/_config.js new file mode 100644 index 00000000000..9bdaef2662d --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_config.js @@ -0,0 +1,18 @@ +const source = Buffer.from('abc'); + +module.exports = { + description: 'warns if multiple files with the same name are emitted', + options: { + input: 'main.js', + plugins: [ + { + buildStart() { + this.emitFile({ type: 'asset', name: 'buildStart', source }); + }, + generateBundle() { + this.emitFile({ type: 'asset', name: 'generateBundle', source }); + } + } + ] + } +}; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js new file mode 100644 index 00000000000..f99c5d38fdf --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js @@ -0,0 +1,9 @@ +define(['exports'], function (exports) { 'use strict'; + + function hi() { return 2 } + + exports.hi = hi; + + Object.defineProperty(exports, '__esModule', { value: true }); + +}); diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile new file mode 100644 index 00000000000..f2ba8f84ab5 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js new file mode 100644 index 00000000000..ce637f0dcc8 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js @@ -0,0 +1,7 @@ +'use strict'; + +Object.defineProperty(exports, '__esModule', { value: true }); + +function hi() { return 2 } + +exports.hi = hi; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile new file mode 100644 index 00000000000..f2ba8f84ab5 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js new file mode 100644 index 00000000000..47c2f790880 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js @@ -0,0 +1,3 @@ +function hi() { return 2 } + +export { hi }; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile new file mode 100644 index 00000000000..f2ba8f84ab5 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js new file mode 100644 index 00000000000..0581de90888 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js @@ -0,0 +1,12 @@ +System.register([], function (exports) { + 'use strict'; + return { + execute: function () { + + exports('hi', hi); + + function hi() { return 2 } + + } + }; +}); diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile new file mode 100644 index 00000000000..f2ba8f84ab5 --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/main.js new file mode 100644 index 00000000000..be03a2f1d9f --- /dev/null +++ b/test/function/samples/emit-file/deduplicate-binary-asset/main.js @@ -0,0 +1 @@ +export function hi() { return 2 } From 7ad10a97d83fb66b0116e0d3c56dc25f257866ff Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 13:21:20 +0100 Subject: [PATCH 05/10] chore: remove option and simplify code --- docs/02-javascript-api.md | 1 - docs/999-big-list-of-options.md | 6 --- src/rollup/types.d.ts | 2 - src/utils/FileEmitter.ts | 46 ++++--------------- src/utils/options/mergeOptions.ts | 1 - src/utils/options/normalizeOutputOptions.ts | 1 - test/cli/samples/watch/bundle-error/main.js | 1 + .../samples/output-options-hook/_config.js | 1 - test/misc/optionList.js | 4 +- 9 files changed, 12 insertions(+), 51 deletions(-) create mode 100644 test/cli/samples/watch/bundle-error/main.js diff --git a/docs/02-javascript-api.md b/docs/02-javascript-api.md index 321d0643c17..c4cc17bfae8 100755 --- a/docs/02-javascript-api.md +++ b/docs/02-javascript-api.md @@ -181,7 +181,6 @@ const outputOptions = { noConflict, preferConst, sanitizeFileName, - deduplicateBinaryAssets, strict, systemNullSetters }; diff --git a/docs/999-big-list-of-options.md b/docs/999-big-list-of-options.md index 3c6ac6bfa20..0eff6b40e3c 100755 --- a/docs/999-big-list-of-options.md +++ b/docs/999-big-list-of-options.md @@ -1468,12 +1468,6 @@ export default { // -> define(['./chunk-or-local-file.js', 'dependency', 'third/dependency'],... ``` -#### output.deduplicateBinaryAssets - -Type: `boolean`
Default: `false` - -Set to `true` to also deduplicate assets when calling `emitFile` with a `Buffer` as source. By default only string sources are deduplicated. - #### output.esModule Type: `boolean | "if-default-prop"`
CLI: `--esModule`/`--no-esModule`
Default: `"if-default-prop"` diff --git a/src/rollup/types.d.ts b/src/rollup/types.d.ts index d0c73229131..70805021277 100644 --- a/src/rollup/types.d.ts +++ b/src/rollup/types.d.ts @@ -632,7 +632,6 @@ export interface OutputOptions { banner?: string | AddonFunction; chunkFileNames?: string | ((chunkInfo: PreRenderedChunk) => string); compact?: boolean; - deduplicateBinaryAssets?: boolean; // only required for bundle.write dir?: string; /** @deprecated Use the "renderDynamicImport" plugin hook instead. */ @@ -686,7 +685,6 @@ export interface NormalizedOutputOptions { banner: AddonFunction; chunkFileNames: string | ((chunkInfo: PreRenderedChunk) => string); compact: boolean; - deduplicateBinaryAssets: boolean; dir: string | undefined; /** @deprecated Use the "renderDynamicImport" plugin hook instead. */ dynamicImportFunction: string | undefined; diff --git a/src/utils/FileEmitter.ts b/src/utils/FileEmitter.ts index 4fb0274bf83..cbbb74d5de8 100644 --- a/src/utils/FileEmitter.ts +++ b/src/utils/FileEmitter.ts @@ -35,7 +35,7 @@ function getSourceHash(source: string | Uint8Array, size?: number): string { .slice(0, Math.max(0, size || defaultHashSize)); } -function prepareAssetFileName( +function pregenerateAssetFileName( name: string | undefined, source: string | Uint8Array, outputOptions: NormalizedOutputOptions @@ -58,17 +58,6 @@ function prepareAssetFileName( return { fileName, sourceHash }; } -function generateAssetFileName( - name: string | undefined, - source: string | Uint8Array, - outputOptions: NormalizedOutputOptions, - bundle: OutputBundleWithPlaceholders, - preparedFileName?: string -): string { - const fileName = preparedFileName || prepareAssetFileName(name, source, outputOptions).fileName; - return makeUnique(fileName, bundle); -} - function reserveFileNameInBundle( fileName: string, { bundle }: FileEmitterOutput, @@ -341,41 +330,24 @@ export class FileEmitter { referenceId: string, { bundle, fileNamesBySource, outputOptions }: FileEmitterOutput ): void { - let preparedFileName: string | undefined; - let sourceHash: string | undefined; - let fileName = consumedFile.fileName; + // Deduplicate assets if an explicit fileName is not provided if (!fileName) { - if (outputOptions.deduplicateBinaryAssets) { - // Reuse source hash if it is going to be needed in the asset file name - const prepared = prepareAssetFileName(consumedFile.name, source, outputOptions); - preparedFileName = prepared.fileName; - sourceHash = prepared.sourceHash || getSourceHash(source); - fileName = fileNamesBySource.get(sourceHash); - } else if (typeof source === 'string') { - fileName = fileNamesBySource.get(source); + const pregenerated = pregenerateAssetFileName(consumedFile.name, source, outputOptions); + // Reuse source hash if it was computed as part of the asset file name + const sourceHash = pregenerated.sourceHash ?? getSourceHash(source); + fileName = fileNamesBySource.get(sourceHash); + if (!fileName) { + fileName = makeUnique(pregenerated.fileName, bundle); + fileNamesBySource.set(sourceHash, fileName); } } - fileName ||= generateAssetFileName( - consumedFile.name, - source, - outputOptions, - bundle, - preparedFileName - ); - // We must not modify the original assets to avoid interaction between outputs const assetWithFileName = { ...consumedFile, fileName, source }; this.filesByReferenceId.set(referenceId, assetWithFileName); - if (sourceHash) { - fileNamesBySource.set(sourceHash, fileName); - } else if (typeof source === 'string') { - fileNamesBySource.set(source, fileName); - } - bundle[fileName] = { fileName, name: consumedFile.name, diff --git a/src/utils/options/mergeOptions.ts b/src/utils/options/mergeOptions.ts index ccd2493d08a..be429cfb598 100644 --- a/src/utils/options/mergeOptions.ts +++ b/src/utils/options/mergeOptions.ts @@ -227,7 +227,6 @@ async function mergeOutputOptions( banner: getOption('banner'), chunkFileNames: getOption('chunkFileNames'), compact: getOption('compact'), - deduplicateBinaryAssets: getOption('deduplicateBinaryAssets'), dir: getOption('dir'), dynamicImportFunction: getOption('dynamicImportFunction'), dynamicImportInCjs: getOption('dynamicImportInCjs'), diff --git a/src/utils/options/normalizeOutputOptions.ts b/src/utils/options/normalizeOutputOptions.ts index 66b97059744..e03e92eba9b 100644 --- a/src/utils/options/normalizeOutputOptions.ts +++ b/src/utils/options/normalizeOutputOptions.ts @@ -45,7 +45,6 @@ export async function normalizeOutputOptions( banner: getAddon(config, 'banner'), chunkFileNames: config.chunkFileNames ?? '[name]-[hash].js', compact, - deduplicateBinaryAssets: config.deduplicateBinaryAssets ?? false, dir: getDir(config, file), dynamicImportFunction: getDynamicImportFunction(config, inputOptions, format), dynamicImportInCjs: config.dynamicImportInCjs ?? true, diff --git a/test/cli/samples/watch/bundle-error/main.js b/test/cli/samples/watch/bundle-error/main.js new file mode 100644 index 00000000000..a4012bff06c --- /dev/null +++ b/test/cli/samples/watch/bundle-error/main.js @@ -0,0 +1 @@ +export default 42; \ No newline at end of file diff --git a/test/function/samples/output-options-hook/_config.js b/test/function/samples/output-options-hook/_config.js index bbcc8cf074c..ed8e82de12c 100644 --- a/test/function/samples/output-options-hook/_config.js +++ b/test/function/samples/output-options-hook/_config.js @@ -24,7 +24,6 @@ module.exports = { assetFileNames: 'assets/[name]-[hash][extname]', chunkFileNames: '[name]-[hash].js', compact: false, - deduplicateBinaryAssets: false, dynamicImportInCjs: true, entryFileNames: '[name].js', esModule: 'if-default-prop', diff --git a/test/misc/optionList.js b/test/misc/optionList.js index f7770c19c85..9cf18ae43ca 100644 --- a/test/misc/optionList.js +++ b/test/misc/optionList.js @@ -1,6 +1,6 @@ exports.input = 'acorn, acornInjectPlugins, cache, context, experimentalCacheExpiry, external, inlineDynamicImports, input, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, moduleContext, onwarn, perf, plugins, preserveEntrySignatures, preserveModules, preserveSymlinks, shimMissingExports, strictDeprecations, treeshake, watch'; exports.flags = - 'acorn, acornInjectPlugins, amd, assetFileNames, banner, bundleConfigAsCjs, c, cache, chunkFileNames, compact, config, configPlugin, context, d, deduplicateBinaryAssets, dir, dynamicImportFunction, dynamicImportInCjs, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalImportAssertions, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, generatedCode, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; + 'acorn, acornInjectPlugins, amd, assetFileNames, banner, bundleConfigAsCjs, c, cache, chunkFileNames, compact, config, configPlugin, context, d, dir, dynamicImportFunction, dynamicImportInCjs, e, entryFileNames, environment, esModule, experimentalCacheExpiry, exports, extend, external, externalImportAssertions, externalLiveBindings, f, failAfterWarnings, file, footer, format, freeze, g, generatedCode, globals, h, hoistTransitiveImports, i, indent, inlineDynamicImports, input, interop, intro, m, makeAbsoluteExternalsRelative, manualChunks, maxParallelFileOps, maxParallelFileReads, minifyInternalExports, moduleContext, n, name, namespaceToStringTag, noConflict, o, onwarn, outro, p, paths, perf, plugin, plugins, preferConst, preserveEntrySignatures, preserveModules, preserveModulesRoot, preserveSymlinks, sanitizeFileName, shimMissingExports, silent, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, stdin, strict, strictDeprecations, systemNullSetters, treeshake, v, validate, w, waitForBundleInput, watch'; exports.output = - 'amd, assetFileNames, banner, chunkFileNames, compact, deduplicateBinaryAssets, dir, dynamicImportFunction, dynamicImportInCjs, entryFileNames, esModule, exports, extend, externalImportAssertions, externalLiveBindings, file, footer, format, freeze, generatedCode, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate'; + 'amd, assetFileNames, banner, chunkFileNames, compact, dir, dynamicImportFunction, dynamicImportInCjs, entryFileNames, esModule, exports, extend, externalImportAssertions, externalLiveBindings, file, footer, format, freeze, generatedCode, globals, hoistTransitiveImports, indent, inlineDynamicImports, interop, intro, manualChunks, minifyInternalExports, name, namespaceToStringTag, noConflict, outro, paths, plugins, preferConst, preserveModules, preserveModulesRoot, sanitizeFileName, sourcemap, sourcemapBaseUrl, sourcemapExcludeSources, sourcemapFile, sourcemapPathTransform, strict, systemNullSetters, validate'; From 12ad13a9d82d9bd4425a9910a705f503338b8a60 Mon Sep 17 00:00:00 2001 From: patak Date: Thu, 10 Nov 2022 13:23:09 +0100 Subject: [PATCH 06/10] docs: update emitFile deduplication note --- docs/05-plugin-development.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/05-plugin-development.md b/docs/05-plugin-development.md index a8c0f3110a5..89839ce7d8a 100644 --- a/docs/05-plugin-development.md +++ b/docs/05-plugin-development.md @@ -732,7 +732,7 @@ If there are no dynamic imports, this will create exactly three chunks where the Note that even though any module id can be used in `implicitlyLoadedAfterOneOf`, Rollup will throw an error if such an id cannot be uniquely associated with a chunk, e.g. because the `id` cannot be reached implicitly or explicitly from the existing static entry points, or because the file is completely tree-shaken. Using only entry points, either defined by the user or of previously emitted chunks, will always work, though. -If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same `string` source even if the `name` does not match. If the source is not a string but a typed array or `Buffer`, no deduplication will occur for performance reasons. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used. +If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same source even if the `name` does not match. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used. #### `this.error` From bacaf626c44b8f0b0e765f4a10ac1ad2e2ebe260 Mon Sep 17 00:00:00 2001 From: patak Date: Fri, 11 Nov 2022 10:24:02 +0100 Subject: [PATCH 07/10] chore: remove missplaced test case --- .../deduplicate-binary-asset/_config.js | 18 ------------------ .../_expected/amd/main.js | 9 --------- .../_expected/amd/myfile | 1 - .../_expected/cjs/main.js | 7 ------- .../_expected/cjs/myfile | 1 - .../_expected/es/main.js | 3 --- .../_expected/es/myfile | 1 - .../_expected/system/main.js | 12 ------------ .../_expected/system/myfile | 1 - .../emit-file/deduplicate-binary-asset/main.js | 1 - 10 files changed, 54 deletions(-) delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_config.js delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile delete mode 100644 test/function/samples/emit-file/deduplicate-binary-asset/main.js diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_config.js b/test/function/samples/emit-file/deduplicate-binary-asset/_config.js deleted file mode 100644 index 9bdaef2662d..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_config.js +++ /dev/null @@ -1,18 +0,0 @@ -const source = Buffer.from('abc'); - -module.exports = { - description: 'warns if multiple files with the same name are emitted', - options: { - input: 'main.js', - plugins: [ - { - buildStart() { - this.emitFile({ type: 'asset', name: 'buildStart', source }); - }, - generateBundle() { - this.emitFile({ type: 'asset', name: 'generateBundle', source }); - } - } - ] - } -}; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js deleted file mode 100644 index f99c5d38fdf..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/main.js +++ /dev/null @@ -1,9 +0,0 @@ -define(['exports'], function (exports) { 'use strict'; - - function hi() { return 2 } - - exports.hi = hi; - - Object.defineProperty(exports, '__esModule', { value: true }); - -}); diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile deleted file mode 100644 index f2ba8f84ab5..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/amd/myfile +++ /dev/null @@ -1 +0,0 @@ -abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js deleted file mode 100644 index ce637f0dcc8..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/main.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -Object.defineProperty(exports, '__esModule', { value: true }); - -function hi() { return 2 } - -exports.hi = hi; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile deleted file mode 100644 index f2ba8f84ab5..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/cjs/myfile +++ /dev/null @@ -1 +0,0 @@ -abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js deleted file mode 100644 index 47c2f790880..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/main.js +++ /dev/null @@ -1,3 +0,0 @@ -function hi() { return 2 } - -export { hi }; diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile deleted file mode 100644 index f2ba8f84ab5..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/es/myfile +++ /dev/null @@ -1 +0,0 @@ -abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js deleted file mode 100644 index 0581de90888..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/main.js +++ /dev/null @@ -1,12 +0,0 @@ -System.register([], function (exports) { - 'use strict'; - return { - execute: function () { - - exports('hi', hi); - - function hi() { return 2 } - - } - }; -}); diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile b/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile deleted file mode 100644 index f2ba8f84ab5..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/_expected/system/myfile +++ /dev/null @@ -1 +0,0 @@ -abc \ No newline at end of file diff --git a/test/function/samples/emit-file/deduplicate-binary-asset/main.js b/test/function/samples/emit-file/deduplicate-binary-asset/main.js deleted file mode 100644 index be03a2f1d9f..00000000000 --- a/test/function/samples/emit-file/deduplicate-binary-asset/main.js +++ /dev/null @@ -1 +0,0 @@ -export function hi() { return 2 } From 9dbffe152fb0fe90ab2768e4843c076e84507d4d Mon Sep 17 00:00:00 2001 From: patak Date: Fri, 11 Nov 2022 10:24:43 +0100 Subject: [PATCH 08/10] test: extend chunking-form deduplicate-assets --- .../emit-file/deduplicate-assets/_config.js | 35 +++++++++++++++++++ .../_expected/amd/assets/buffer-d0ca8c2a.txt | 1 + .../amd/assets/otherBuffer-e8d9b528.txt | 1 + .../_expected/amd/named/buffer.txt | 1 + .../_expected/cjs/assets/buffer-d0ca8c2a.txt | 1 + .../cjs/assets/otherBuffer-e8d9b528.txt | 1 + .../_expected/cjs/named/buffer.txt | 1 + .../_expected/es/assets/buffer-d0ca8c2a.txt | 1 + .../es/assets/otherBuffer-e8d9b528.txt | 1 + .../_expected/es/named/buffer.txt | 1 + .../system/assets/buffer-d0ca8c2a.txt | 1 + .../system/assets/otherBuffer-e8d9b528.txt | 1 + .../_expected/system/named/buffer.txt | 1 + 13 files changed, 47 insertions(+) create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/buffer-d0ca8c2a.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherBuffer-e8d9b528.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/buffer.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/buffer-d0ca8c2a.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherBuffer-e8d9b528.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/buffer.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/buffer-d0ca8c2a.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherBuffer-e8d9b528.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/buffer.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/buffer-d0ca8c2a.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherBuffer-e8d9b528.txt create mode 100644 test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/buffer.txt diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_config.js b/test/chunking-form/samples/emit-file/deduplicate-assets/_config.js index 1f37410f516..41d3beb4ca0 100644 --- a/test/chunking-form/samples/emit-file/deduplicate-assets/_config.js +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_config.js @@ -5,10 +5,45 @@ module.exports = { plugins: { buildStart() { this.emitFile({ type: 'asset', name: 'string.txt', source: 'string' }); + this.emitFile({ type: 'asset', name: 'stringSameSource.txt', source: 'string' }); + this.emitFile({ + type: 'asset', + name: 'sameStringAsBuffer.txt', + source: Buffer.from('string') // Test cross Buffer/string deduplication + }); + // Different string source this.emitFile({ type: 'asset', name: 'otherString.txt', source: 'otherString' }); + const bufferSource = () => Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72]); + this.emitFile({ + type: 'asset', + name: 'buffer.txt', + source: bufferSource() + }); + this.emitFile({ + type: 'asset', + name: 'bufferSameSource.txt', + source: bufferSource() + }); + this.emitFile({ + type: 'asset', + name: 'sameBufferAsString.txt', + source: bufferSource().toString() // Test cross Buffer/string deduplication + }); + // Different buffer source + this.emitFile({ + type: 'asset', + name: 'otherBuffer.txt', + source: Buffer.from('otherBuffer') + }); + // specific file names will not be deduplicated this.emitFile({ type: 'asset', fileName: 'named/string.txt', source: 'named' }); + this.emitFile({ + type: 'asset', + fileName: 'named/buffer.txt', + source: bufferSource() + }); return null; } } diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/buffer-d0ca8c2a.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/buffer-d0ca8c2a.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/buffer-d0ca8c2a.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherBuffer-e8d9b528.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherBuffer-e8d9b528.txt new file mode 100644 index 00000000000..54eb26bae5c --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherBuffer-e8d9b528.txt @@ -0,0 +1 @@ +otherBuffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/buffer.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/buffer.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/buffer.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/buffer-d0ca8c2a.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/buffer-d0ca8c2a.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/buffer-d0ca8c2a.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherBuffer-e8d9b528.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherBuffer-e8d9b528.txt new file mode 100644 index 00000000000..54eb26bae5c --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherBuffer-e8d9b528.txt @@ -0,0 +1 @@ +otherBuffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/buffer.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/buffer.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/buffer.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/buffer-d0ca8c2a.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/buffer-d0ca8c2a.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/buffer-d0ca8c2a.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherBuffer-e8d9b528.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherBuffer-e8d9b528.txt new file mode 100644 index 00000000000..54eb26bae5c --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherBuffer-e8d9b528.txt @@ -0,0 +1 @@ +otherBuffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/buffer.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/buffer.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/buffer.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/buffer-d0ca8c2a.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/buffer-d0ca8c2a.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/buffer-d0ca8c2a.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherBuffer-e8d9b528.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherBuffer-e8d9b528.txt new file mode 100644 index 00000000000..54eb26bae5c --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherBuffer-e8d9b528.txt @@ -0,0 +1 @@ +otherBuffer \ No newline at end of file diff --git a/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/buffer.txt b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/buffer.txt new file mode 100644 index 00000000000..ea75f1e9c19 --- /dev/null +++ b/test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/buffer.txt @@ -0,0 +1 @@ +buffer \ No newline at end of file From aa9a4e1c0a0664144f2718a89c9be7b99afa3961 Mon Sep 17 00:00:00 2001 From: patak Date: Fri, 11 Nov 2022 10:25:18 +0100 Subject: [PATCH 09/10] chore: simplify sourceHash generation --- src/utils/FileEmitter.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/utils/FileEmitter.ts b/src/utils/FileEmitter.ts index cbbb74d5de8..a14a55fe4b1 100644 --- a/src/utils/FileEmitter.ts +++ b/src/utils/FileEmitter.ts @@ -39,7 +39,7 @@ function pregenerateAssetFileName( name: string | undefined, source: string | Uint8Array, outputOptions: NormalizedOutputOptions -): { fileName: string; sourceHash: string | undefined } { +): { fileName: string; sourceHash: string } { let sourceHash: string | undefined; const emittedName = outputOptions.sanitizeFileName(name || 'asset'); const fileName = renderNamePattern( @@ -55,6 +55,7 @@ function pregenerateAssetFileName( emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length)) } ); + sourceHash ??= getSourceHash(source); return { fileName, sourceHash }; } @@ -335,12 +336,10 @@ export class FileEmitter { // Deduplicate assets if an explicit fileName is not provided if (!fileName) { const pregenerated = pregenerateAssetFileName(consumedFile.name, source, outputOptions); - // Reuse source hash if it was computed as part of the asset file name - const sourceHash = pregenerated.sourceHash ?? getSourceHash(source); - fileName = fileNamesBySource.get(sourceHash); + fileName = fileNamesBySource.get(pregenerated.sourceHash); if (!fileName) { fileName = makeUnique(pregenerated.fileName, bundle); - fileNamesBySource.set(sourceHash, fileName); + fileNamesBySource.set(pregenerated.sourceHash, fileName); } } From 7e41f02915eb2a7a7b293947a325a05594d8188a Mon Sep 17 00:00:00 2001 From: patak Date: Fri, 11 Nov 2022 21:12:02 +0100 Subject: [PATCH 10/10] refactor: precompute hash then slice --- src/utils/FileEmitter.ts | 59 ++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/utils/FileEmitter.ts b/src/utils/FileEmitter.ts index a14a55fe4b1..8478b8ab199 100644 --- a/src/utils/FileEmitter.ts +++ b/src/utils/FileEmitter.ts @@ -28,35 +28,34 @@ import { extname } from './path'; import { isPathFragment } from './relativeId'; import { makeUnique, renderNamePattern } from './renderNamePattern'; -function getSourceHash(source: string | Uint8Array, size?: number): string { - return createHash() - .update(source) - .digest('hex') - .slice(0, Math.max(0, size || defaultHashSize)); +function getSourceHash(source: string | Uint8Array): string { + return createHash().update(source).digest('hex'); } -function pregenerateAssetFileName( +function generateAssetFileName( name: string | undefined, source: string | Uint8Array, - outputOptions: NormalizedOutputOptions -): { fileName: string; sourceHash: string } { - let sourceHash: string | undefined; + sourceHash: string, + outputOptions: NormalizedOutputOptions, + bundle: OutputBundleWithPlaceholders +): string { const emittedName = outputOptions.sanitizeFileName(name || 'asset'); - const fileName = renderNamePattern( - typeof outputOptions.assetFileNames === 'function' - ? outputOptions.assetFileNames({ name, source, type: 'asset' }) - : outputOptions.assetFileNames, - 'output.assetFileNames', - { - ext: () => extname(emittedName).slice(1), - extname: () => extname(emittedName), - hash: size => (sourceHash = getSourceHash(source, size)), - name: () => - emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length)) - } + return makeUnique( + renderNamePattern( + typeof outputOptions.assetFileNames === 'function' + ? outputOptions.assetFileNames({ name, source, type: 'asset' }) + : outputOptions.assetFileNames, + 'output.assetFileNames', + { + ext: () => extname(emittedName).slice(1), + extname: () => extname(emittedName), + hash: size => sourceHash.slice(0, Math.max(0, size || defaultHashSize)), + name: () => + emittedName.slice(0, Math.max(0, emittedName.length - extname(emittedName).length)) + } + ), + bundle ); - sourceHash ??= getSourceHash(source); - return { fileName, sourceHash }; } function reserveFileNameInBundle( @@ -335,11 +334,17 @@ export class FileEmitter { // Deduplicate assets if an explicit fileName is not provided if (!fileName) { - const pregenerated = pregenerateAssetFileName(consumedFile.name, source, outputOptions); - fileName = fileNamesBySource.get(pregenerated.sourceHash); + const sourceHash = getSourceHash(source); + fileName = fileNamesBySource.get(sourceHash); if (!fileName) { - fileName = makeUnique(pregenerated.fileName, bundle); - fileNamesBySource.set(pregenerated.sourceHash, fileName); + fileName = generateAssetFileName( + consumedFile.name, + source, + sourceHash, + outputOptions, + bundle + ); + fileNamesBySource.set(sourceHash, fileName); } }