Add --lazy option to support building on demand in development (#5835)

packages/core/core/src/AssetGraph.js

@@ -277,8 +277,8 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
     return !defer;
   }
 
-  // Dependency: mark parent Asset <- AssetGroup with hasDeferred false
-  markParentsWithHasDeferred(node: DependencyNode) {
+  // Dependency: mark parent Asset <- AssetGroup with hasDeferred true
+  markParentsWithHasDeferred(node: AssetGraphNode) {
     this.traverseAncestors(node, (_node, _, actions) => {
       if (_node.type === 'asset') {
         _node.hasDeferred = true;
@@ -292,7 +292,7 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
   }
 
   // AssetGroup: update hasDeferred of all parent Dependency <- Asset <- AssetGroup
-  unmarkParentsWithHasDeferred(node: AssetGroupNode) {
+  unmarkParentsWithHasDeferred(node: AssetGraphNode) {
     this.traverseAncestors(node, (_node, ctx, actions) => {
       if (_node.type === 'asset') {
         let hasDeferred = this.getNodesConnectedFrom(_node).some(_childNode =>

packages/core/core/src/PackagerRunner.js

@@ -119,8 +119,27 @@ export default class PackagerRunner {
     |} = {};
     let writeEarlyPromises = {};
     let hashRefToNameHash = new Map();
-    // skip inline bundles, they will be processed via the parent bundle
-    let bundles = bundleGraph.getBundles().filter(bundle => !bundle.isInline);
+    let bundles = bundleGraph.getBundles().filter(bundle => {
+      // Do not package and write placeholder bundles to disk. We just
+      // need to update the name so other bundles can reference it.
+      if (bundle.isPlaceholder) {
+        let hash = bundle.id.slice(-8);
+        hashRefToNameHash.set(bundle.hashReference, hash);
+        bundle.filePath = nullthrows(bundle.filePath).replace(
+          bundle.hashReference,
+          hash,
+        );
+        bundle.name = nullthrows(bundle.name).replace(
+          bundle.hashReference,
+          hash,
+        );
+        return false;
+      }
+
+      // skip inline bundles, they will be processed via the parent bundle
+      return !bundle.isInline;
+    });
+
     try {
       await Promise.all(
         bundles.map(async bundle => {
@@ -716,15 +735,11 @@ function assignComplexNameHashes(
       continue;
     }
 
-    let includedBundles = [
-      ...getBundlesIncludedInHash(bundle.id, bundleInfoMap),
-    ];
-
     hashRefToNameHash.set(
       bundle.hashReference,
       options.shouldContentHash
         ? md5FromString(
-            includedBundles
+            [...getBundlesIncludedInHash(bundle.id, bundleInfoMap)]
               .map(bundleId => bundleInfoMap[bundleId].hash)
               .join(':'),
           ).slice(-8)

packages/core/core/src/Parcel.js

@@ -71,6 +71,7 @@ export default class Parcel {
   > */;
   #watcherSubscription /*: ?AsyncSubscription*/;
   #watcherCount /*: number*/ = 0;
+  #requestedAssetIds /*: Set<string>*/ = new Set();
 
   isProfiling /*: boolean */;
 
@@ -183,16 +184,20 @@ export default class Parcel {
     await this.#farm.callAllWorkers('clearConfigCache', []);
   }
 
-  async _startNextBuild() {
+  async _startNextBuild(): Promise<?BuildEvent> {
     this.#watchAbortController = new AbortController();
     await this.#farm.callAllWorkers('clearConfigCache', []);
 
     try {
+      let buildEvent = await this._build({
+        signal: this.#watchAbortController.signal,
+      });
+
       this.#watchEvents.emit({
-        buildEvent: await this._build({
-          signal: this.#watchAbortController.signal,
-        }),
+        buildEvent,
       });
+
+      return buildEvent;
     } catch (err) {
       // Ignore BuildAbortErrors and only emit critical errors.
       if (!(err instanceof BuildAbortError)) {
@@ -273,14 +278,21 @@ export default class Parcel {
       });
       let request = createAssetGraphRequest({
         name: 'Main',
-        entries: nullthrows(this.#resolvedOptions).entries,
+        entries: options.entries,
         optionsRef: this.#optionsRef,
+        shouldBuildLazily: options.shouldBuildLazily,
+        requestedAssetIds: this.#requestedAssetIds,
       }); // ? should we create this on every build?
       let {
         assetGraph,
         changedAssets,
         assetRequests,
-      } = await this.#requestTracker.runRequest(request);
+      } = await this.#requestTracker.runRequest(request, {
+        force: options.shouldBuildLazily && this.#requestedAssetIds.size > 0,
+      });
+
+      this.#requestedAssetIds.clear();
+
       dumpGraphToGraphViz(assetGraph, 'MainAssetGraph');
 
       let [
@@ -311,6 +323,41 @@ export default class Parcel {
         ),
         bundleGraph: new BundleGraph(bundleGraph, NamedBundle.get, options),
         buildTime: Date.now() - startTime,
+        requestBundle: async bundle => {
+          let bundleNode = bundleGraph._graph.getNode(bundle.id);
+          invariant(bundleNode?.type === 'bundle', 'Bundle does not exist');
+
+          if (!bundleNode.value.isPlaceholder) {
+            // Nothing to do.
+            return {
+              type: 'buildSuccess',
+              changedAssets: new Map(),
+              bundleGraph: event.bundleGraph,
+              buildTime: 0,
+              requestBundle: event.requestBundle,
+            };
+          }
+
+          for (let assetId of bundleNode.value.entryAssetIds) {
+            this.#requestedAssetIds.add(assetId);
+          }
+
+          if (this.#watchQueue.getNumWaiting() === 0) {
+            if (this.#watchAbortController) {
+              this.#watchAbortController.abort();
+            }
+
+            this.#watchQueue.add(() => this._startNextBuild());
+          }
+
+          let results = await this.#watchQueue.run();
+          let result = results.filter(Boolean).pop();
+          if (result.type === 'buildFailure') {
+            throw new BuildError(result.diagnostics);
+          }
+
+          return result;
+        },
       };
 
       await this.#reporterRunner.report(event);

packages/core/core/src/RequestTracker.js

@@ -518,6 +518,7 @@ export class RequestGraph extends Graph<
   }
 
   respondToFSEvents(events: Array<Event>): boolean {
+    let didInvalidate = false;
     for (let {path: filePath, type} of events) {
       let node = this.getNode(filePath);
 
@@ -527,12 +528,14 @@ export class RequestGraph extends Graph<
       if (node && (type === 'create' || type === 'update')) {
         let nodes = this.getNodesConnectedTo(node, 'invalidated_by_update');
         for (let connectedNode of nodes) {
+          didInvalidate = true;
           this.invalidateNode(connectedNode, FILE_UPDATE);
         }
 
         if (type === 'create') {
           let nodes = this.getNodesConnectedTo(node, 'invalidated_by_create');
           for (let connectedNode of nodes) {
+            didInvalidate = true;
             this.invalidateNode(connectedNode, FILE_CREATE);
           }
         }
@@ -550,6 +553,7 @@ export class RequestGraph extends Graph<
           });
 
           if (above.length > 0) {
+            didInvalidate = true;
             this.invalidateFileNameNode(fileNameNode, filePath, above);
           }
         }
@@ -564,6 +568,7 @@ export class RequestGraph extends Graph<
               'invalidated_by_create',
             );
             for (let connectedNode of connectedNodes) {
+              didInvalidate = true;
               this.invalidateNode(connectedNode, FILE_CREATE);
             }
           }
@@ -573,12 +578,13 @@ export class RequestGraph extends Graph<
           node,
           'invalidated_by_delete',
         )) {
+          didInvalidate = true;
           this.invalidateNode(connectedNode, FILE_DELETE);
         }
       }
     }
 
-    return this.invalidNodeIds.size > 0;
+    return didInvalidate && this.invalidNodeIds.size > 0;
   }
 }
 

packages/core/core/src/public/MutableBundleGraph.js

@@ -159,6 +159,13 @@ export default class MutableBundleGraph extends BundleGraph<IBundle>
     );
     this.#bundlePublicIds.add(publicId);
 
+    let isPlaceholder = false;
+    if (entryAsset) {
+      let entryAssetNode = this.#graph._graph.getNode(entryAsset.id);
+      invariant(entryAssetNode?.type === 'asset', 'Entry asset does not exist');
+      isPlaceholder = entryAssetNode.requested === false;
+    }
+
     let bundleNode = {
       type: 'bundle',
       id: bundleId,
@@ -176,6 +183,7 @@ export default class MutableBundleGraph extends BundleGraph<IBundle>
         isEntry: opts.isEntry,
         isInline: opts.isInline,
         isSplittable: opts.isSplittable ?? entryAsset?.isSplittable,
+        isPlaceholder,
         target,
         name: null,
         displayName: null,

packages/core/core/src/public/PluginOptions.js

@@ -52,6 +52,10 @@ export default class PluginOptions implements IPluginOptions {
     return this.#options.serveOptions;
   }
 
+  get shouldBuildLazily(): boolean {
+    return this.#options.shouldBuildLazily;
+  }
+
   get shouldAutoInstall(): boolean {
     return this.#options.shouldAutoInstall;
   }

packages/core/core/src/requests/AssetGraphRequest.js

@@ -44,6 +44,8 @@ type AssetGraphRequestInput = {|
   assetGroups?: Array<AssetGroup>,
   optionsRef: SharedReference,
   name: string,
+  shouldBuildLazily?: boolean,
+  requestedAssetIds?: Set<string>,
 |};
 
 type RunInput = {|
@@ -98,9 +100,18 @@ export class AssetGraphBuilder {
   name: string;
   assetRequests: Array<AssetGroup> = [];
   cacheKey: string;
+  shouldBuildLazily: boolean;
+  requestedAssetIds: Set<string>;
 
   constructor({input, prevResult, api, options}: RunInput) {
-    let {entries, assetGroups, optionsRef, name} = input;
+    let {
+      entries,
+      assetGroups,
+      optionsRef,
+      name,
+      requestedAssetIds,
+      shouldBuildLazily,
+    } = input;
     let assetGraph = prevResult?.assetGraph ?? new AssetGraph();
     assetGraph.setRootConnections({
       entries,
@@ -111,6 +122,8 @@ export class AssetGraphBuilder {
     this.options = options;
     this.api = api;
     this.name = name;
+    this.requestedAssetIds = requestedAssetIds ?? new Set();
+    this.shouldBuildLazily = shouldBuildLazily ?? false;
 
     this.cacheKey = md5FromOrderedObject({
       parcelVersion: PARCEL_VERSION,
@@ -152,7 +165,7 @@ export class AssetGraphBuilder {
       for (let child of this.assetGraph.getNodesConnectedFrom(node)) {
         if (
           (!visited.has(child.id) || child.hasDeferred) &&
-          this.assetGraph.shouldVisitChild(node, child)
+          this.shouldVisitChild(node, child)
         ) {
           visited.add(child.id);
           visit(child);
@@ -201,6 +214,38 @@ export class AssetGraphBuilder {
     };
   }
 
+  shouldVisitChild(node: AssetGraphNode, child: AssetGraphNode): boolean {
+    if (this.shouldBuildLazily) {
+      if (node.type === 'asset' && child.type === 'dependency') {
+        if (this.requestedAssetIds.has(node.value.id)) {
+          node.requested = true;
+        } else if (!node.requested) {
+          let isAsyncChild = this.assetGraph
+            .getIncomingDependencies(node.value)
+            .every(dep => dep.isEntry || dep.isAsync);
+          if (isAsyncChild) {
+            node.requested = false;
+          } else {
+            delete node.requested;
+          }
+        }
+
+        let previouslyDeferred = child.deferred;
+        child.deferred = node.requested === false;
+
+        if (!previouslyDeferred && child.deferred) {
+          this.assetGraph.markParentsWithHasDeferred(child);
+        } else if (previouslyDeferred && !child.deferred) {
+          this.assetGraph.unmarkParentsWithHasDeferred(child);
+        }
+
+        return !child.deferred;
+      }
+    }
+
+    return this.assetGraph.shouldVisitChild(node, child);
+  }
+
   propagateSymbols() {
     // Propagate the requested symbols down from the root to the leaves
     this.propagateSymbolsDown((assetNode, incomingDeps, outgoingDeps) => {

packages/core/core/src/resolveOptions.js

@@ -82,6 +82,13 @@ export default async function resolveOptions(
       ? path.resolve(inputCwd, initialOptions?.defaultTargetOptions?.distDir)
       : undefined;
 
+  let shouldBuildLazily = initialOptions.shouldBuildLazily ?? false;
+  let shouldContentHash =
+    initialOptions.shouldContentHash ?? initialOptions.mode === 'production';
+  if (shouldBuildLazily && shouldContentHash) {
+    throw new Error('Lazy bundling does not work with content hashing');
+  }
+
   return {
     config: initialOptions.config,
     defaultConfig: initialOptions.defaultConfig,
@@ -99,8 +106,8 @@ export default async function resolveOptions(
     mode,
     shouldAutoInstall: initialOptions.shouldAutoInstall ?? false,
     hmrOptions: initialOptions.hmrOptions ?? null,
-    shouldContentHash:
-      initialOptions.shouldContentHash ?? initialOptions.mode === 'production',
+    shouldBuildLazily,
+    shouldContentHash,
     serveOptions: initialOptions.serveOptions
       ? {
           ...initialOptions.serveOptions,

packages/core/core/src/types.js

@@ -172,6 +172,7 @@ export type ParcelOptions = {|
   hmrOptions: ?HMROptions,
   shouldContentHash: boolean,
   serveOptions: ServerOptions | false,
+  shouldBuildLazily: boolean,
   shouldAutoInstall: boolean,
   logLevel: LogLevel,
   projectRoot: FilePath,
@@ -220,6 +221,7 @@ export type AssetNode = {|
   hasDeferred?: boolean,
   usedSymbolsDownDirty: boolean,
   usedSymbolsUpDirty: boolean,
+  requested?: boolean,
 |};
 
 export type DependencyNode = {|
@@ -407,6 +409,7 @@ export type Bundle = {|
   isEntry: ?boolean,
   isInline: ?boolean,
   isSplittable: ?boolean,
+  isPlaceholder?: boolean,
   target: Target,
   filePath: ?FilePath,
   name: ?string,

packages/core/core/test/test-utils.js

@@ -23,6 +23,7 @@ export const DEFAULT_OPTIONS: ParcelOptions = {
   shouldAutoInstall: false,
   hmrOptions: undefined,
   shouldContentHash: true,
+  shouldBuildLazily: false,
   serveOptions: false,
   mode: 'development',
   env: {},