Skip to content

Commit

Permalink
feat(splitting): support minChunkSize option
Browse files Browse the repository at this point in the history
  • Loading branch information
Yidong Li committed Aug 9, 2023
1 parent 813fb3a commit 8851bd5
Show file tree
Hide file tree
Showing 12 changed files with 178 additions and 1 deletion.
1 change: 1 addition & 0 deletions cmd/esbuild/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var helpText = func(colors logger.Colors) string {
--serve=... Start a local HTTP server on this host:port for outputs
--sourcemap Emit a source map
--splitting Enable code splitting (currently only for esm)
--min-chunk-size Control min chunk source size for code splitting (currently only for js)
--target=... Environment target (e.g. es2017, chrome58, firefox57,
safari11, edge16, node10, ie9, opera45, default esnext)
--watch Watch mode: rebuild on file system changes (stops when
Expand Down
31 changes: 31 additions & 0 deletions internal/bundler_tests/bundler_splitting_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,37 @@ func TestSplittingDynamicAndNotDynamicCommonJSIntoES6(t *testing.T) {
})
}

func TestSplittingWithMinChunkSize(t *testing.T) {
splitting_suite.expectBundled(t, bundled{
files: map[string]string{
"/a.js": `
import * as ns from './common_mini'
import * as nsl from './common_large'
export let a = 'a' + ns.foo
export let aa = 'a' + nsl.bar
`,
"/b.js": `
import * as ns from './common_mini'
export let b = 'b' + ns.foo
`,
"/c.js": `
import * as ns from './common_large'
export let b = 'b' + ns.bar
`,
"/common_mini.js": `export let foo = 123`,
"/common_large.js": `export let bar = 1234`,
},
entryPaths: []string{"/a.js", "/b.js", "/c.js"},
options: config.Options{
Mode: config.ModeBundle,
CodeSplitting: true,
MinChunkSize: 21,
OutputFormat: config.FormatESModule,
AbsOutputDir: "/out",
},
})
}

func TestSplittingAssignToLocal(t *testing.T) {
splitting_suite.expectBundled(t, bundled{
files: map[string]string{
Expand Down
47 changes: 47 additions & 0 deletions internal/bundler_tests/snapshots/snapshots_splitting.txt
Original file line number Diff line number Diff line change
Expand Up @@ -622,3 +622,50 @@ export {
a,
b
};

================================================================================
TestSplittingWithMinChunkSize
---------- /out/a.js ----------
import {
bar
} from "./chunk-RIRTUAAF.js";

// common_mini.js
var foo = 123;

// a.js
var a = "a" + foo;
var aa = "a" + bar;
export {
a,
aa
};

---------- /out/b.js ----------
// common_mini.js
var foo = 123;

// b.js
var b = "b" + foo;
export {
b
};

---------- /out/c.js ----------
import {
bar
} from "./chunk-RIRTUAAF.js";

// c.js
var b = "b" + bar;
export {
b
};

---------- /out/chunk-RIRTUAAF.js ----------
// common_large.js
var bar = 1234;

export {
bar
};
1 change: 1 addition & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,7 @@ type Options struct {
MinifySyntax bool
ProfilerNames bool
CodeSplitting bool
MinChunkSize int
WatchMode bool
AllowOverwrite bool
LegalComments LegalComments
Expand Down
3 changes: 3 additions & 0 deletions internal/graph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ type LinkerFile struct {
// This is true if this file has been marked as live by the tree shaking
// algorithm.
IsLive bool

// If true, this part can't be extract to cross chunk dependencies.
SplitOff bool
}

func (f *LinkerFile) IsEntryPoint() bool {
Expand Down
48 changes: 47 additions & 1 deletion internal/linker/linker.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ type chunkInfo struct {
// For code splitting
crossChunkImports []chunkImport

// This is the total size of input files
chunkSize int

// This is the representation-specific information
chunkRepr chunkRepr

Expand Down Expand Up @@ -962,6 +965,13 @@ func (c *linkerContext) computeCrossChunkDependencies() {
// chunk. In that case this will overwrite the same value below which
// is fine.
for _, declared := range part.DeclaredSymbols {
// If SourceIndex is marked as SplitOff, it means the symbol
// is from different entry chunk, skip chunkIndex written to escape
// DATA RACE
if declared.Ref.SourceIndex < uint32(len(c.graph.Files)) && c.graph.Files[declared.Ref.SourceIndex].SplitOff {
continue
}

if declared.IsTopLevel {
c.graph.Symbols.Get(declared.Ref).ChunkIndex = ast.MakeIndex32(uint32(chunkIndex))
}
Expand All @@ -973,6 +983,13 @@ func (c *linkerContext) computeCrossChunkDependencies() {
for ref := range part.SymbolUses {
symbol := c.graph.Symbols.Get(ref)

// Ignore all symbols if bound file is SplitOff
// If SourceIndex is larger than graph.Files, it means the symbol
// is from a generated chunk and SplitOff check can be skipped
if symbol.Link.SourceIndex < uint32(len(c.graph.Files)) && c.graph.Files[symbol.Link.SourceIndex].SplitOff {
continue
}

// Ignore unbound symbols, which don't have declarations
if symbol.Kind == ast.SymbolUnbound {
continue
Expand Down Expand Up @@ -3676,6 +3693,7 @@ func (c *linkerContext) computeChunks() {

jsChunks := make(map[string]chunkInfo)
cssChunks := make(map[string]chunkInfo)
entryKeys := make([]string, 0, len(c.graph.EntryPoints()))

// Create chunks for entry points
for i, entryPoint := range c.graph.EntryPoints() {
Expand All @@ -3691,8 +3709,10 @@ func (c *linkerContext) computeChunks() {
isEntryPoint: true,
sourceIndex: entryPoint.SourceIndex,
entryPointBit: uint(i),
chunkSize: 0,
filesWithPartsInChunk: make(map[uint32]bool),
}
entryKeys = append(entryKeys, key)

switch file.InputFile.Repr.(type) {
case *graph.JSRepr:
Expand Down Expand Up @@ -3723,6 +3743,7 @@ func (c *linkerContext) computeChunks() {
externalImportsInOrder: externalOrder,
filesInChunkInOrder: internalOrder,
},
chunkSize: 0,
}
chunkRepr.hasCSSChunk = true
}
Expand Down Expand Up @@ -3750,13 +3771,38 @@ func (c *linkerContext) computeChunks() {
chunk.entryBits = file.EntryBits
chunk.filesWithPartsInChunk = make(map[uint32]bool)
chunk.chunkRepr = &chunkReprJS{}
chunk.chunkSize = 0
jsChunks[key] = chunk
}
chunk.filesWithPartsInChunk[uint32(sourceIndex)] = true
}
}
}

// remove chunks by user configuration. This matters because auto code splitting
// may generate lots of mini chunks
for key := range jsChunks {
jsChunk := jsChunks[key]
// calculate each jsChunk's size
for sourceIdx := range jsChunk.filesWithPartsInChunk {
jsChunk.chunkSize += len(c.graph.Files[sourceIdx].InputFile.Source.Contents)
}
// If current js chunk is smaller than the minimal chunkSize config, mark this file as SplitOff
// and move it to the entryChunks it belongs to
if !jsChunk.isEntryPoint && jsChunk.chunkSize < c.options.MinChunkSize {
for _, entryKey := range entryKeys {
entryChunk := jsChunks[entryKey]
if jsChunk.entryBits.HasBit(entryChunk.entryPointBit) {
for sourceIdx := range jsChunk.filesWithPartsInChunk {
c.graph.Files[sourceIdx].SplitOff = true
entryChunk.filesWithPartsInChunk[sourceIdx] = true
}
}
}
delete(jsChunks, key)
}
}

// Sort the chunks for determinism. This matters because we use chunk indices
// as sorting keys in a few places.
sortedChunks := make([]chunkInfo, 0, len(jsChunks)+len(cssChunks))
Expand Down Expand Up @@ -3970,7 +4016,7 @@ func (c *linkerContext) findImportedPartsInJSOrder(chunk *chunkInfo) (js []uint3
file := &c.graph.Files[sourceIndex]

if repr, ok := file.InputFile.Repr.(*graph.JSRepr); ok {
isFileInThisChunk := chunk.entryBits.Equals(file.EntryBits)
isFileInThisChunk := file.SplitOff || chunk.entryBits.Equals(file.EntryBits)

// Wrapped files can't be split because they are all inside the wrapper
canFileBeSplit := repr.Meta.Wrap == graph.WrapNone
Expand Down
2 changes: 2 additions & 0 deletions lib/shared/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ function flagsForBuildOptions(
let sourcemap = getFlag(options, keys, 'sourcemap', mustBeStringOrBoolean)
let bundle = getFlag(options, keys, 'bundle', mustBeBoolean)
let splitting = getFlag(options, keys, 'splitting', mustBeBoolean)
let minChunkSize = getFlag(options, keys, 'minChunkSize', mustBeInteger)
let preserveSymlinks = getFlag(options, keys, 'preserveSymlinks', mustBeBoolean)
let metafile = getFlag(options, keys, 'metafile', mustBeBoolean)
let outfile = getFlag(options, keys, 'outfile', mustBeString)
Expand Down Expand Up @@ -284,6 +285,7 @@ function flagsForBuildOptions(
if (bundle) flags.push('--bundle')
if (allowOverwrite) flags.push('--allow-overwrite')
if (splitting) flags.push('--splitting')
if (minChunkSize) flags.push(`--min-chunk-size=${minChunkSize}`)
if (preserveSymlinks) flags.push('--preserve-symlinks')
if (metafile) flags.push(`--metafile`)
if (outfile) flags.push(`--outfile=${outfile}`)
Expand Down
2 changes: 2 additions & 0 deletions lib/shared/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ export interface BuildOptions extends CommonOptions {
bundle?: boolean
/** Documentation: https://esbuild.github.io/api/#splitting */
splitting?: boolean
/** */
minChunkSize?: number
/** Documentation: https://esbuild.github.io/api/#preserve-symlinks */
preserveSymlinks?: boolean
/** Documentation: https://esbuild.github.io/api/#outfile */
Expand Down
1 change: 1 addition & 0 deletions pkg/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ type BuildOptions struct {
Bundle bool // Documentation: https://esbuild.github.io/api/#bundle
PreserveSymlinks bool // Documentation: https://esbuild.github.io/api/#preserve-symlinks
Splitting bool // Documentation: https://esbuild.github.io/api/#splitting
MinChunkSize int // Documentation:
Outfile string // Documentation: https://esbuild.github.io/api/#outfile
Metafile bool // Documentation: https://esbuild.github.io/api/#metafile
Outdir string // Documentation: https://esbuild.github.io/api/#outdir
Expand Down
1 change: 1 addition & 0 deletions pkg/api/api_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -1303,6 +1303,7 @@ func validateBuildOptions(
TreeShaking: validateTreeShaking(buildOpts.TreeShaking, buildOpts.Bundle, buildOpts.Format),
GlobalName: validateGlobalName(log, buildOpts.GlobalName),
CodeSplitting: buildOpts.Splitting,
MinChunkSize: buildOpts.MinChunkSize,
OutputFormat: validateFormat(buildOpts.Format),
AbsOutputFile: validatePath(log, realFS, buildOpts.Outfile, "outfile path"),
AbsOutputDir: validatePath(log, realFS, buildOpts.Outdir, "outdir path"),
Expand Down
13 changes: 13 additions & 0 deletions pkg/cli/cli_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,19 @@ func parseOptionsImpl(
buildOpts.Splitting = value
}

case strings.HasPrefix(arg, "--min-chunk-size="):
value := arg[len("--min-chunk-size="):]
minChunkSize, err := strconv.Atoi(value)
if err != nil || minChunkSize < 0 {
return parseOptionsExtras{}, cli_helpers.MakeErrorWithNote(
fmt.Sprintf("Invalid value %q in %q", value, arg),
"The min chunk size must be a non-negative integer.",
)
}
if buildOpts != nil {
buildOpts.MinChunkSize = minChunkSize
}

case isBoolFlag(arg, "--allow-overwrite") && buildOpts != nil:
if value, err := parseBoolFlag(arg, true); err != nil {
return parseOptionsExtras{}, err
Expand Down
29 changes: 29 additions & 0 deletions scripts/end-to-end-tests.js
Original file line number Diff line number Diff line change
Expand Up @@ -7038,6 +7038,35 @@ tests.push(
`,
}),

// Code splitting via minChunkSize control
test([
'a.js', 'b.js', 'c.js', '--splitting',
'--outdir=out', '--format=esm', '--bundle', '--min-chunk-size=22'
], {
'a.js': `
import * as ns from './common_mini'
import * as nsl from './common_large'
export let a = 'a' + ns.foo
export let aa = 'a' + nsl.bar
`,
'b.js': `
import * as ns from './common_mini'
export let b = 'b' + ns.foo
`,
'c.js': `
import * as ns from './common_large'
export let c = 'c' + ns.bar
`,
'common_mini.js': `export let foo = 123`,
'common_large.js': `export let bar = 1234`,
'node.js': `
import {a, aa} from './out/a.js'
import {b} from './out/b.js'
import {c} from './out/c.js'
if (a !== 'a123' || aa !== 'a1234' || b !== 'b123' || c !== 'c1234') throw 'fail'
`,
}),

// Code splitting via ES6 module double-imported with sync and async imports
test(['a.js', '--outdir=out', '--splitting', '--format=esm', '--bundle'], {
'a.js': `
Expand Down

0 comments on commit 8851bd5

Please sign in to comment.