From f0200b1dc0c62335a4dcf3cff0b377789af49fdc Mon Sep 17 00:00:00 2001 From: bluwy Date: Mon, 7 Feb 2022 00:26:35 +0800 Subject: [PATCH 1/5] feat(optimize): support custom extensions --- .../src/node/optimizer/esbuildDepPlugin.ts | 14 +++-- packages/vite/src/node/optimizer/index.ts | 62 +++++++++++++------ packages/vite/src/node/optimizer/scan.ts | 14 ++++- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index 6c778e2c8bf8d3..ccbb4bc69d71ca 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -1,5 +1,5 @@ import path from 'path' -import type { Loader, Plugin, ImportKind } from 'esbuild' +import type { Plugin, ImportKind } from 'esbuild' import { KNOWN_ASSET_TYPES } from '../constants' import type { ResolvedConfig } from '..' import { @@ -40,6 +40,12 @@ export function esbuildDepPlugin( config: ResolvedConfig, ssr?: boolean ): Plugin { + const allExternalTypes = config.optimizeDeps.supportedExtensions + ? externalTypes.filter( + (type) => !config.optimizeDeps.supportedExtensions?.includes('.' + type) + ) + : externalTypes + // default resolver which prefers ESM const _resolve = config.createResolver({ asSrc: false }) @@ -74,7 +80,7 @@ export function esbuildDepPlugin( // externalize assets and commonly known non-js file types build.onResolve( { - filter: new RegExp(`\\.(` + externalTypes.join('|') + `)(\\?.*)?$`) + filter: new RegExp(`\\.(` + allExternalTypes.join('|') + `)(\\?.*)?$`) }, async ({ path: id, importer, kind }) => { const resolved = await resolve(id, importer, kind) @@ -181,10 +187,8 @@ export function esbuildDepPlugin( } } - let ext = path.extname(entryFile).slice(1) - if (ext === 'mjs') ext = 'js' return { - loader: ext as Loader, + loader: 'js', contents, resolveDir: root } diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 6102e832841a89..49aba03e47e26d 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -79,6 +79,12 @@ export interface DepOptimizationOptions { * @deprecated use `esbuildOptions.keepNames` */ keepNames?: boolean + /** + * List of file extensions that can be optimized. A corresponding esbuild + * plugin must exist to handle the specific extension. + * @experimental + */ + supportedExtensions?: string[] } export interface DepOptimizationMetadata { @@ -244,29 +250,47 @@ export async function optimizeDeps( for (const id in deps) { const flatId = flattenId(id) const filePath = (flatIdDeps[flatId] = deps[id]) - const entryContent = fs.readFileSync(filePath, 'utf-8') let exportsData: ExportsData - try { - exportsData = parse(entryContent) as ExportsData - } catch { - debug( - `Unable to parse dependency: ${id}. Trying again with a JSX transform.` + if ( + config.optimizeDeps.supportedExtensions?.some((ext) => + filePath.endsWith(ext) ) - const transformed = await transformWithEsbuild(entryContent, filePath, { - loader: 'jsx' + ) { + // For custom supported extensions, build the entry file to transform it into JS, + // and then parse with es-module-lexer. Note that the `bundle` option is not `true`, + // so only the entry file is being transformed. + const result = await build({ + ...esbuildOptions, + plugins, + entryPoints: [filePath], + write: false, + format: 'esm' }) - // Ensure that optimization won't fail by defaulting '.js' to the JSX parser. - // This is useful for packages such as Gatsby. - esbuildOptions.loader = { - '.js': 'jsx', - ...esbuildOptions.loader + exportsData = parse(result.outputFiles[0].text) as ExportsData + } else { + const entryContent = fs.readFileSync(filePath, 'utf-8') + try { + exportsData = parse(entryContent) as ExportsData + } catch { + debug( + `Unable to parse dependency: ${id}. Trying again with a JSX transform.` + ) + const transformed = await transformWithEsbuild(entryContent, filePath, { + loader: 'jsx' + }) + // Ensure that optimization won't fail by defaulting '.js' to the JSX parser. + // This is useful for packages such as Gatsby. + esbuildOptions.loader = { + '.js': 'jsx', + ...esbuildOptions.loader + } + exportsData = parse(transformed.code) as ExportsData } - exportsData = parse(transformed.code) as ExportsData - } - for (const { ss, se } of exportsData[0]) { - const exp = entryContent.slice(ss, se) - if (/export\s+\*\s+from/.test(exp)) { - exportsData.hasReExports = true + for (const { ss, se } of exportsData[0]) { + const exp = entryContent.slice(ss, se) + if (/export\s+\*\s+from/.test(exp)) { + exportsData.hasReExports = true + } } } idToExports[id] = exportsData diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index 9ba65d23684305..9bcd773969ab4a 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -181,6 +181,10 @@ function esbuildScanPlugin( '@vite/env' ] + const canOptimize = (id: string) => + OPTIMIZABLE_ENTRY_RE.test(id) || + !!config.optimizeDeps.supportedExtensions?.some((ext) => id.endsWith(ext)) + const externalUnlessEntry = ({ path }: { path: string }) => ({ path, external: !entries.includes(path) @@ -218,8 +222,14 @@ function esbuildScanPlugin( // html types: extract script contents ----------------------------------- build.onResolve({ filter: htmlTypesRE }, async ({ path, importer }) => { + const resolved = await resolve(path, importer) + if (!resolved) return + // It is possible for the scanner to scan html types in node_modules. + // If we can optimize this html type, skip it so it's handled by the + // bare import resolve, and recorded as optimization dep. + if (resolved.includes('node_modules') && canOptimize(resolved)) return return { - path: await resolve(path, importer), + path: resolved, namespace: 'html' } }) @@ -340,7 +350,7 @@ function esbuildScanPlugin( } if (resolved.includes('node_modules') || include?.includes(id)) { // dependency or forced included, externalize and stop crawling - if (OPTIMIZABLE_ENTRY_RE.test(resolved)) { + if (canOptimize(resolved)) { depImports[id] = resolved } return externalUnlessEntry({ path: id }) From 0a2a3a0e065495d3eaa1090263163f3c02a0abd5 Mon Sep 17 00:00:00 2001 From: bluwy Date: Thu, 10 Feb 2022 15:07:59 +0800 Subject: [PATCH 2/5] chore: add test --- .../__tests__/optimize-deps.spec.ts | 4 ++ .../optimize-deps/dep-not-js/foo.js | 1 + .../optimize-deps/dep-not-js/index.notjs | 5 +++ .../optimize-deps/dep-not-js/package.json | 6 +++ packages/playground/optimize-deps/index.html | 6 +++ .../playground/optimize-deps/package.json | 1 + .../playground/optimize-deps/vite.config.js | 38 +++++++++++++++++++ pnpm-lock.yaml | 12 ++++-- 8 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 packages/playground/optimize-deps/dep-not-js/foo.js create mode 100644 packages/playground/optimize-deps/dep-not-js/index.notjs create mode 100644 packages/playground/optimize-deps/dep-not-js/package.json diff --git a/packages/playground/optimize-deps/__tests__/optimize-deps.spec.ts b/packages/playground/optimize-deps/__tests__/optimize-deps.spec.ts index 706998c288fa50..aa8fd9aaa51be8 100644 --- a/packages/playground/optimize-deps/__tests__/optimize-deps.spec.ts +++ b/packages/playground/optimize-deps/__tests__/optimize-deps.spec.ts @@ -62,6 +62,10 @@ test('import * from optimized dep', async () => { expect(await page.textContent('.import-star')).toMatch(`[success]`) }) +test('import from dep with .notjs files', async () => { + expect(await page.textContent('.not-js')).toMatch(`[success]`) +}) + test('dep with css import', async () => { expect(await getColor('h1')).toBe('red') }) diff --git a/packages/playground/optimize-deps/dep-not-js/foo.js b/packages/playground/optimize-deps/dep-not-js/foo.js new file mode 100644 index 00000000000000..3d809780820c7b --- /dev/null +++ b/packages/playground/optimize-deps/dep-not-js/foo.js @@ -0,0 +1 @@ +export const foo = '[success] imported from .notjs file' diff --git a/packages/playground/optimize-deps/dep-not-js/index.notjs b/packages/playground/optimize-deps/dep-not-js/index.notjs new file mode 100644 index 00000000000000..d9b6d151390a51 --- /dev/null +++ b/packages/playground/optimize-deps/dep-not-js/index.notjs @@ -0,0 +1,5 @@ + +import { foo } from './foo' +const notjsValue = foo +export notjsValue + diff --git a/packages/playground/optimize-deps/dep-not-js/package.json b/packages/playground/optimize-deps/dep-not-js/package.json new file mode 100644 index 00000000000000..39ebafb6217b6e --- /dev/null +++ b/packages/playground/optimize-deps/dep-not-js/package.json @@ -0,0 +1,6 @@ +{ + "name": "dep-not-js", + "private": true, + "version": "1.0.0", + "main": "index.notjs" +} diff --git a/packages/playground/optimize-deps/index.html b/packages/playground/optimize-deps/index.html index dfb274656f1baf..ae2b4adf6462c0 100644 --- a/packages/playground/optimize-deps/index.html +++ b/packages/playground/optimize-deps/index.html @@ -38,6 +38,9 @@

Optimizing force included dep even when it's linked

import * as ...

+

Import from dependency with .notjs files

+
+

Dep w/ special file format supported via plugins

@@ -70,6 +73,9 @@

Nested include

text('.import-star', `[success] ${keys.join(', ')}`) } + import { notjsValue } from 'dep-not-js' + text('.not-js', notjsValue) + import { createApp } from 'vue' import { createStore } from 'vuex' if (typeof createApp === 'function' && typeof createStore === 'function') { diff --git a/packages/playground/optimize-deps/package.json b/packages/playground/optimize-deps/package.json index 0606343e0dce3c..277075f3c8933c 100644 --- a/packages/playground/optimize-deps/package.json +++ b/packages/playground/optimize-deps/package.json @@ -17,6 +17,7 @@ "dep-esbuild-plugin-transform": "file:./dep-esbuild-plugin-transform", "dep-linked": "link:./dep-linked", "dep-linked-include": "link:./dep-linked-include", + "dep-not-js": "file:./dep-not-js", "lodash-es": "^4.17.21", "nested-exclude": "file:./nested-exclude", "phoenix": "^1.6.2", diff --git a/packages/playground/optimize-deps/vite.config.js b/packages/playground/optimize-deps/vite.config.js index 45a50aaf85ede6..216f45d340f385 100644 --- a/packages/playground/optimize-deps/vite.config.js +++ b/packages/playground/optimize-deps/vite.config.js @@ -1,3 +1,4 @@ +const fs = require('fs') const vue = require('@vitejs/plugin-vue') /** @@ -36,6 +37,7 @@ module.exports = { plugins: [ vue(), + notjs(), // for axios request test { name: 'mock', @@ -48,3 +50,39 @@ module.exports = { } ] } + +// Handles .notjs file, basically remove wrapping and tags +function notjs() { + return { + name: 'notjs', + config() { + return { + optimizeDeps: { + supportedExtensions: ['.notjs'], + esbuildOptions: { + plugins: [ + { + name: 'esbuild-notjs', + setup(build) { + build.onLoad({ filter: /\.notjs$/ }, ({ path }) => { + let contents = fs.readFileSync(path, 'utf-8') + contents = contents + .replace('', '') + .replace('', '') + return { contents, loader: 'js' } + }) + } + } + ] + } + } + } + }, + transform(code, id) { + if (id.endsWith('.notjs')) { + code = code.replace('', '').replace('', '') + return { code } + } + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b40b226c292e61..97f4c7e45698ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -278,6 +278,7 @@ importers: dep-esbuild-plugin-transform: file:./dep-esbuild-plugin-transform dep-linked: link:./dep-linked dep-linked-include: link:./dep-linked-include + dep-not-js: file:./dep-not-js lodash-es: ^4.17.21 nested-exclude: file:./nested-exclude phoenix: ^1.6.2 @@ -294,6 +295,7 @@ importers: dep-esbuild-plugin-transform: link:dep-esbuild-plugin-transform dep-linked: link:dep-linked dep-linked-include: link:dep-linked-include + dep-not-js: link:dep-not-js lodash-es: 4.17.21 nested-exclude: link:nested-exclude phoenix: 1.6.5 @@ -326,6 +328,9 @@ importers: dependencies: react: 17.0.2 + packages/playground/optimize-deps/dep-not-js: + specifiers: {} + packages/playground/optimize-deps/nested-exclude: specifiers: nested-include: link:./nested-include @@ -5157,7 +5162,6 @@ packages: /graceful-fs/4.2.9: resolution: {integrity: sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==} - dev: true /handlebars/4.7.7: resolution: {integrity: sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==} @@ -6148,7 +6152,7 @@ packages: '@types/node': 16.11.22 chalk: 4.1.2 ci-info: 3.3.0 - graceful-fs: 4.2.8 + graceful-fs: 4.2.9 picomatch: 2.3.0 dev: true @@ -6333,7 +6337,7 @@ packages: dependencies: universalify: 2.0.0 optionalDependencies: - graceful-fs: 4.2.8 + graceful-fs: 4.2.9 /jsonparse/1.3.1: resolution: {integrity: sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=} @@ -6388,7 +6392,7 @@ packages: tslib: 2.3.1 optionalDependencies: errno: 0.1.8 - graceful-fs: 4.2.8 + graceful-fs: 4.2.9 image-size: 0.5.5 make-dir: 2.1.0 mime: 1.6.0 From fa45463a934e30f7d4dfb49b279e182f6b438edf Mon Sep 17 00:00:00 2001 From: bluwy Date: Thu, 10 Feb 2022 15:11:38 +0800 Subject: [PATCH 3/5] fix(scan): handle supported non-html extensions --- packages/vite/src/node/optimizer/scan.ts | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index 9bcd773969ab4a..29339877863ad1 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -181,7 +181,7 @@ function esbuildScanPlugin( '@vite/env' ] - const canOptimize = (id: string) => + const isOptimizable = (id: string) => OPTIMIZABLE_ENTRY_RE.test(id) || !!config.optimizeDeps.supportedExtensions?.some((ext) => id.endsWith(ext)) @@ -227,7 +227,7 @@ function esbuildScanPlugin( // It is possible for the scanner to scan html types in node_modules. // If we can optimize this html type, skip it so it's handled by the // bare import resolve, and recorded as optimization dep. - if (resolved.includes('node_modules') && canOptimize(resolved)) return + if (resolved.includes('node_modules') && isOptimizable(resolved)) return return { path: resolved, namespace: 'html' @@ -350,17 +350,19 @@ function esbuildScanPlugin( } if (resolved.includes('node_modules') || include?.includes(id)) { // dependency or forced included, externalize and stop crawling - if (canOptimize(resolved)) { + if (isOptimizable(resolved)) { depImports[id] = resolved } return externalUnlessEntry({ path: id }) - } else { + } else if (isScannable(resolved)) { const namespace = htmlTypesRE.test(resolved) ? 'html' : undefined // linked package, keep crawling return { path: path.resolve(resolved), namespace } + } else { + return externalUnlessEntry({ path: id }) } } else { missing[id] = normalizePath(importer) @@ -406,7 +408,7 @@ function esbuildScanPlugin( // use vite resolver to support urls and omitted extensions const resolved = await resolve(id, importer) if (resolved) { - if (shouldExternalizeDep(resolved, id)) { + if (shouldExternalizeDep(resolved, id) || !isScannable(resolved)) { return externalUnlessEntry({ path: id }) } @@ -509,10 +511,7 @@ function extractImportPaths(code: string) { return js } -export function shouldExternalizeDep( - resolvedId: string, - rawId: string -): boolean { +function shouldExternalizeDep(resolvedId: string, rawId: string): boolean { // not a valid file path if (!path.isAbsolute(resolvedId)) { return true @@ -521,9 +520,9 @@ export function shouldExternalizeDep( if (resolvedId === rawId || resolvedId.includes('\0')) { return true } - // resolved is not a scannable type - if (!JS_TYPES_RE.test(resolvedId) && !htmlTypesRE.test(resolvedId)) { - return true - } return false } + +function isScannable(id: string): boolean { + return JS_TYPES_RE.test(id) || htmlTypesRE.test(id) +} From 7395900931559e7f2c38b17ff311bdcd6c811ce5 Mon Sep 17 00:00:00 2001 From: bluwy Date: Thu, 10 Feb 2022 15:31:50 +0800 Subject: [PATCH 4/5] chore: typo --- packages/playground/optimize-deps/dep-not-js/index.notjs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/playground/optimize-deps/dep-not-js/index.notjs b/packages/playground/optimize-deps/dep-not-js/index.notjs index d9b6d151390a51..b4ef3a6936a797 100644 --- a/packages/playground/optimize-deps/dep-not-js/index.notjs +++ b/packages/playground/optimize-deps/dep-not-js/index.notjs @@ -1,5 +1,4 @@ import { foo } from './foo' -const notjsValue = foo -export notjsValue +export const notjsValue = foo From 1c25715e4cb7f62cac3ef0df0fab3bc8d83430c8 Mon Sep 17 00:00:00 2001 From: bluwy Date: Sun, 13 Feb 2022 21:34:08 +0800 Subject: [PATCH 5/5] chore: rename extensions option --- packages/playground/optimize-deps/vite.config.js | 2 +- packages/vite/src/node/optimizer/esbuildDepPlugin.ts | 5 +++-- packages/vite/src/node/optimizer/index.ts | 12 ++++++------ packages/vite/src/node/optimizer/scan.ts | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/playground/optimize-deps/vite.config.js b/packages/playground/optimize-deps/vite.config.js index 216f45d340f385..314d1c49cbf645 100644 --- a/packages/playground/optimize-deps/vite.config.js +++ b/packages/playground/optimize-deps/vite.config.js @@ -58,7 +58,7 @@ function notjs() { config() { return { optimizeDeps: { - supportedExtensions: ['.notjs'], + extensions: ['.notjs'], esbuildOptions: { plugins: [ { diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index ccbb4bc69d71ca..3ff86c213a54a2 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -40,9 +40,10 @@ export function esbuildDepPlugin( config: ResolvedConfig, ssr?: boolean ): Plugin { - const allExternalTypes = config.optimizeDeps.supportedExtensions + // remove optimizable extensions from `externalTypes` list + const allExternalTypes = config.optimizeDeps.extensions ? externalTypes.filter( - (type) => !config.optimizeDeps.supportedExtensions?.includes('.' + type) + (type) => !config.optimizeDeps.extensions?.includes('.' + type) ) : externalTypes diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 49aba03e47e26d..c759d71ab5ed65 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -82,9 +82,13 @@ export interface DepOptimizationOptions { /** * List of file extensions that can be optimized. A corresponding esbuild * plugin must exist to handle the specific extension. + * + * By default, Vite can optimize `.mjs`, `.js`, and `.ts` files. This option + * allows specifying additional extensions. + * * @experimental */ - supportedExtensions?: string[] + extensions?: string[] } export interface DepOptimizationMetadata { @@ -251,11 +255,7 @@ export async function optimizeDeps( const flatId = flattenId(id) const filePath = (flatIdDeps[flatId] = deps[id]) let exportsData: ExportsData - if ( - config.optimizeDeps.supportedExtensions?.some((ext) => - filePath.endsWith(ext) - ) - ) { + if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) { // For custom supported extensions, build the entry file to transform it into JS, // and then parse with es-module-lexer. Note that the `bundle` option is not `true`, // so only the entry file is being transformed. diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index b2b7dc28a33736..b0f5f7985b0f78 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -183,7 +183,7 @@ function esbuildScanPlugin( const isOptimizable = (id: string) => OPTIMIZABLE_ENTRY_RE.test(id) || - !!config.optimizeDeps.supportedExtensions?.some((ext) => id.endsWith(ext)) + !!config.optimizeDeps.extensions?.some((ext) => id.endsWith(ext)) const externalUnlessEntry = ({ path }: { path: string }) => ({ path,