add --packages=external for the node platform

fix #1958
fix #1975
fix #2164
fix #2246
fix #2542

CHANGELOG.md

@@ -2,6 +2,42 @@
 
 ## Unreleased
 
+* Make it easy to exclude all packages from a bundle ([#1958](https://github.com/evanw/esbuild/issues/1958), [#1975](https://github.com/evanw/esbuild/issues/1975), [#2164](https://github.com/evanw/esbuild/issues/2164), [#2246](https://github.com/evanw/esbuild/issues/2246), [#2542](https://github.com/evanw/esbuild/issues/2542))
+
+    When bundling for node, it's often necessary to exclude npm packages from the bundle since they weren't designed with esbuild bundling in mind and don't work correctly after being bundled. For example, they may use `__dirname` and run-time file system calls to load files, which doesn't work after bundling with esbuild. Or they may compile a native `.node` extension that has similar expectations about the layout of the file system that are no longer true after bundling (even if the `.node` extension is copied next to the bundle).
+
+    The way to get this to work with esbuild is to use the `--external:` flag. For example, the [`fsevents`](https://www.npmjs.com/package/fsevents) package contains a native `.node` extension and shouldn't be bundled. To bundle code that uses it, you can pass `--external:fsevents` to esbuild to exclude it from your bundle. You will then need to ensure that the `fsevents` package is still present when you run your bundle (e.g. by publishing your bundle to npm as a package with a dependency on `fsevents`).
+
+    It was possible to automatically do this for all of your dependencies, but it was inconvenient. You had to write some code that read your `package.json` file and passed the keys of the `dependencies`, `devDependencies`, `peerDependencies`, and/or `optionalDependencies` maps to esbuild as external packages (either that or write a plugin to mark all package paths as external). Previously esbuild's recommendation for making this easier was to do `--external:./node_modules/*` (added in version 0.14.13). However, this was a bad idea because it caused compatibility problems with many node packages as it caused esbuild to mark the post-resolve path as external instead of the pre-resolve path. Doing that could break packages that are published as both CommonJS and ESM if esbuild's bundler is also used to do a module format conversion.
+
+    With this release, you can now do the following to automatically exclude all packages from your bundle:
+
+    * CLI:
+
+        ```
+        esbuild --bundle --packages=external
+        ```
+
+    * JS:
+
+        ```js
+        esbuild.build({
+          bundle: true,
+          packages: 'external',
+        })
+        ```
+
+    * Go:
+
+        ```go
+        api.Build(api.BuildOptions{
+          Bundle:   true,
+          Packages: api.PackagesExternal,
+        })
+        ```
+
+    Doing `--external:./node_modules/*` is still possible and still has the same behavior, but is no longer recommended. I recommend that you use the new `packages` feature instead.
+
 * Fix some subtle bugs with tagged template literals
 
     This release fixes a bug where minification could incorrectly change the value of `this` within tagged template literal function calls:

cmd/esbuild/main.go

@@ -43,6 +43,7 @@ var helpText = func(colors logger.Colors) string {
   --minify              Minify the output (sets all --minify-* flags)
   --outdir=...          The output directory (for multiple entry points)
   --outfile=...         The output file (for one entry point)
+  --packages=...        Set to "external" to avoid bundling any package
   --platform=...        Platform target (browser | node | neutral,
                         default browser)
   --serve=...           Start a local HTTP server on this host:port for outputs

internal/bundler/bundler_default_test.go

@@ -7079,3 +7079,27 @@ NOTE: You can either keep the import assertion and only use the "default" import
 `,
 	})
 }
+
+func TestExternalPackages(t *testing.T) {
+	loader_suite.expectBundled(t, bundled{
+		files: map[string]string{
+			"/entry.js": `
+				import 'pkg1'
+				import './file'
+				import './node_modules/pkg2/index.js'
+			`,
+			"/file.js": `
+				console.log('file')
+			`,
+			"/node_modules/pkg2/index.js": `
+				console.log('pkg2')
+			`,
+		},
+		entryPaths: []string{"/entry.js"},
+		options: config.Options{
+			Mode:             config.ModeBundle,
+			AbsOutputFile:    "/out.js",
+			ExternalPackages: true,
+		},
+	})
+}

internal/bundler/snapshots/snapshots_loader.txt

@@ -57,6 +57,18 @@ a:after {
   content: "entry2";
 }
 
+================================================================================
+TestExternalPackages
+---------- /out.js ----------
+// entry.js
+import "pkg1";
+
+// file.js
+console.log("file");
+
+// node_modules/pkg2/index.js
+console.log("pkg2");
+
 ================================================================================
 TestIndirectRequireMessage
 ---------- /out/array.js ----------

internal/config/config.go

@@ -271,6 +271,7 @@ type Options struct {
 	Conditions       []string
 	AbsNodePaths     []string // The "NODE_PATH" variable from Node.js
 	ExternalSettings ExternalSettings
+	ExternalPackages bool
 	PackageAliases   map[string]string
 
 	AbsOutputFile      string

internal/resolver/resolver.go

@@ -359,6 +359,19 @@ func (rr *resolver) Resolve(sourceDir string, importPath string, kind ast.Import
 		}, debugMeta
 	}
 
+	// "import 'pkg'" when all packages are external (vs. "import './pkg'")
+	if r.options.ExternalPackages && IsPackagePath(importPath) && !r.fs.IsAbs(importPath) {
+		if r.debugLogs != nil {
+			r.debugLogs.addNote("Marking this path as external because it's a package path")
+		}
+
+		r.flushDebugLogs(flushDueToSuccess)
+		return &ResolveResult{
+			PathPair:   PathPair{Primary: logger.Path{Text: importPath}},
+			IsExternal: true,
+		}, debugMeta
+	}
+
 	// "import fs from 'fs'"
 	if r.options.Platform == config.PlatformNode && BuiltInNodeModules[importPath] {
 		if r.debugLogs != nil {

lib/shared/common.ts

@@ -261,6 +261,7 @@ function flagsForBuildOptions(
   let mainFields = getFlag(options, keys, 'mainFields', mustBeArray);
   let conditions = getFlag(options, keys, 'conditions', mustBeArray);
   let external = getFlag(options, keys, 'external', mustBeArray);
+  let packages = getFlag(options, keys, 'packages', mustBeString);
   let alias = getFlag(options, keys, 'alias', mustBeObject);
   let loader = getFlag(options, keys, 'loader', mustBeObject);
   let outExtension = getFlag(options, keys, 'outExtension', mustBeObject);
@@ -302,6 +303,7 @@ function flagsForBuildOptions(
   if (outdir) flags.push(`--outdir=${outdir}`);
   if (outbase) flags.push(`--outbase=${outbase}`);
   if (tsconfig) flags.push(`--tsconfig=${tsconfig}`);
+  if (packages) flags.push(`--packages=${packages}`);
   if (resolveExtensions) {
     let values: string[] = [];
     for (let value of resolveExtensions) {

lib/shared/types.ts

@@ -98,6 +98,8 @@ export interface BuildOptions extends CommonOptions {
   outbase?: string;
   /** Documentation: https://esbuild.github.io/api/#external */
   external?: string[];
+  /** Documentation: https://esbuild.github.io/api/#packages */
+  packages?: 'external';
   /** Documentation: https://esbuild.github.io/api/#alias */
   alias?: Record<string, string>;
   /** Documentation: https://esbuild.github.io/api/#loader */

pkg/api/api.go

@@ -165,6 +165,13 @@ const (
 	FormatESModule
 )
 
+type Packages uint8
+
+const (
+	PackagesDefault Packages = iota
+	PackagesExternal
+)
+
 type Engine struct {
 	Name    EngineName
 	Version string
@@ -299,6 +306,7 @@ type BuildOptions struct {
 	Platform          Platform          // Documentation: https://esbuild.github.io/api/#platform
 	Format            Format            // Documentation: https://esbuild.github.io/api/#format
 	External          []string          // Documentation: https://esbuild.github.io/api/#external
+	Packages          Packages          // Documentation: https://esbuild.github.io/api/#packages
 	Alias             map[string]string // Documentation: https://esbuild.github.io/api/#alias
 	MainFields        []string          // Documentation: https://esbuild.github.io/api/#main-fields
 	Conditions        []string          // Documentation: https://esbuild.github.io/api/#conditions

pkg/api/api_impl.go

@@ -1002,6 +1002,7 @@ func rebuildImpl(
 		ExtensionToLoader:     validateLoaders(log, buildOpts.Loader),
 		ExtensionOrder:        validateResolveExtensions(log, buildOpts.ResolveExtensions),
 		ExternalSettings:      validateExternals(log, realFS, buildOpts.External),
+		ExternalPackages:      buildOpts.Packages == PackagesExternal,
 		PackageAliases:        validateAlias(log, realFS, buildOpts.Alias),
 		TsConfigOverride:      validatePath(log, realFS, buildOpts.Tsconfig, "tsconfig path"),
 		MainFields:            buildOpts.MainFields,

pkg/cli/cli_impl.go

@@ -596,6 +596,19 @@ func parseOptionsImpl(
 				transformOpts.Format = format
 			}
 
+		case strings.HasPrefix(arg, "--packages=") && buildOpts != nil:
+			value := arg[len("--packages="):]
+			var packages api.Packages
+			if value == "external" {
+				packages = api.PackagesExternal
+			} else {
+				return parseOptionsExtras{}, cli_helpers.MakeErrorWithNote(
+					fmt.Sprintf("Invalid value %q in %q", value, arg),
+					"The only valid value is \"external\".",
+				)
+			}
+			buildOpts.Packages = packages
+
 		case strings.HasPrefix(arg, "--external:") && buildOpts != nil:
 			buildOpts.External = append(buildOpts.External, arg[len("--external:"):])
 
@@ -825,6 +838,7 @@ func parseOptionsImpl(
 				"outbase":            true,
 				"outdir":             true,
 				"outfile":            true,
+				"packages":           true,
 				"platform":           true,
 				"preserve-symlinks":  true,
 				"public-path":        true,

scripts/js-api-tests.js

@@ -2216,6 +2216,38 @@ require("/assets/file.png");
 `)
   },
 
+  async externalPackages({ esbuild, testDir }) {
+    const input = path.join(testDir, 'in.js')
+    const pkgPath = path.join(testDir, 'node_modules', 'pkg', 'path.js')
+    const dirPath = path.join(testDir, 'dir', 'path.js')
+    await mkdirAsync(path.dirname(pkgPath), { recursive: true })
+    await mkdirAsync(path.dirname(dirPath), { recursive: true })
+    await writeFileAsync(input, `
+      import 'pkg/path.js'
+      import './dir/path.js'
+      import 'before/alias'
+    `)
+    await writeFileAsync(pkgPath, `console.log('pkg')`)
+    await writeFileAsync(dirPath, `console.log('dir')`)
+    const { outputFiles } = await esbuild.build({
+      entryPoints: [input],
+      write: false,
+      bundle: true,
+      packages: 'external',
+      format: 'esm',
+      alias: { 'before': 'after' },
+    })
+    assert.strictEqual(outputFiles[0].text, `// scripts/.js-api-tests/externalPackages/in.js
+import "pkg/path.js";
+
+// scripts/.js-api-tests/externalPackages/dir/path.js
+console.log("dir");
+
+// scripts/.js-api-tests/externalPackages/in.js
+import "after/alias";
+`)
+  },
+
   async errorInvalidExternalWithTwoWildcards({ esbuild }) {
     try {
       await esbuild.build({