V2 scope hoisting (#2967)

* WIP: v2 scope hoisting

* Support sideEffects flag

* Don't edit parcel 1

* Fix flow errors

* Type safe traversal filtering

* Run tests faster

* Base child assets on the same id base as parents

* Fix flow errors

* Only enable scope hoisting in production mode by default

* Fix lint

* Clean up

* Shake exports with pure property assignments (#2909)

* Send BundleGraph to packagers

* Lint

* Fix flow

* Clear scope cache before crawling

* Fix shake

* Define __esModule interop flag when requiring ES module from CommonJS

* Replace module.require in scope hoisting

* Fix assigning to exports from inside a function in scope hoisting

* Format

packages/bundlers/default/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["@parcel/babel-preset"]
+}

packages/core/core/src/Asset.js

@@ -15,6 +15,7 @@ import type {
   PackageJSON,
   SourceMap,
   Stats,
+  Symbol,
   TransformerResult
 } from '@parcel/types';
 
@@ -33,6 +34,7 @@ import Dependency from './Dependency';
 type AssetOptions = {|
   id?: string,
   hash?: ?string,
+  idBase?: string,
   filePath: FilePath,
   type: string,
   content?: Blob,
@@ -46,7 +48,9 @@ type AssetOptions = {|
   outputHash?: string,
   env: Environment,
   meta?: Meta,
-  stats: Stats
+  stats: Stats,
+  symbols?: Map<Symbol, Symbol> | Array<[Symbol, Symbol]>,
+  sideEffects?: boolean
 |};
 
 type SerializedOptions = {|
@@ -60,6 +64,7 @@ type SerializedOptions = {|
 export default class Asset {
   id: string;
   hash: ?string;
+  idBase: string;
   filePath: FilePath;
   type: string;
   ast: ?AST;
@@ -74,13 +79,16 @@ export default class Asset {
   stats: Stats;
   content: Blob;
   contentKey: ?string;
+  symbols: Map<Symbol, Symbol>;
+  sideEffects: boolean;
 
   constructor(options: AssetOptions) {
+    this.idBase = options.idBase != null ? options.idBase : options.filePath;
     this.id =
       options.id != null
         ? options.id
         : md5FromString(
-            options.filePath + options.type + JSON.stringify(options.env)
+            this.idBase + options.type + JSON.stringify(options.env)
           );
     this.hash = options.hash;
     this.filePath = options.filePath;
@@ -100,6 +108,8 @@ export default class Asset {
     this.env = options.env;
     this.meta = options.meta || {};
     this.stats = options.stats;
+    this.symbols = new Map(options.symbols || []);
+    this.sideEffects = options.sideEffects != null ? options.sideEffects : true;
   }
 
   serialize(): SerializedOptions {
@@ -116,7 +126,9 @@ export default class Asset {
       env: this.env,
       meta: this.meta,
       stats: this.stats,
-      contentKey: this.contentKey
+      contentKey: this.contentKey,
+      symbols: [...this.symbols],
+      sideEffects: this.sideEffects
     };
   }
 
@@ -237,8 +249,12 @@ export default class Asset {
       env: this.env.merge(env),
       sourcePath: this.filePath
     });
-
-    this.dependencies.set(dep.id, dep);
+    let existing = this.dependencies.get(dep.id);
+    if (existing) {
+      existing.merge(dep);
+    } else {
+      this.dependencies.set(dep.id, dep);
+    }
     return dep.id;
   }
 
@@ -282,6 +298,7 @@ export default class Asset {
     }
 
     let asset = new Asset({
+      idBase: this.idBase,
       hash,
       filePath: this.filePath,
       type: result.type,
@@ -295,7 +312,10 @@ export default class Asset {
       stats: {
         time: 0,
         size
-      }
+      },
+      symbols: new Map([...this.symbols, ...(result.symbols || [])]),
+      sideEffects:
+        result.sideEffects != null ? result.sideEffects : this.sideEffects
     });
 
     let dependencies = result.dependencies;

packages/core/core/src/AssetGraph.js

@@ -13,14 +13,17 @@ import type {
   Dependency as IDependency,
   File,
   FilePath,
-  GraphTraversalCallback,
   Target,
+  GraphVisitor,
+  Symbol,
+  SymbolResolution,
   TransformerRequest
 } from '@parcel/types';
 
 import type Asset from './Asset';
 
 import invariant from 'assert';
+import nullthrows from 'nullthrows';
 import Graph from './Graph';
 import {md5FromString} from '@parcel/utils';
 import Dependency from './Dependency';
@@ -75,6 +78,9 @@ const getDepNodesFromGraph = (
   );
 };
 
+const invertMap = <K, V>(map: Map<K, V>): Map<V, K> =>
+  new Map([...map].map(([key, val]) => [val, key]));
+
 type DepUpdates = {|
   newRequest?: TransformerRequest,
   prunedFiles: Array<File>
@@ -104,6 +110,7 @@ type AssetGraphOpts = {|
 export default class AssetGraph extends Graph<AssetGraphNode> {
   incompleteNodes: Map<NodeId, AssetGraphNode> = new Map();
   invalidNodes: Map<NodeId, AssetGraphNode> = new Map();
+  deferredNodes: Set<NodeId> = new Set();
 
   initializeGraph({
     entries,
@@ -164,9 +171,37 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
     let requestNode = nodeFromTransformerRequest(req);
     let {added, removed} = this.replaceNodesConnectedTo(depNode, [requestNode]);
 
+    // Defer transforming this dependency if it is marked as weak, there are no side effects,
+    // and no re-exported symbols are used by ancestor dependencies.
+    // This helps with performance building large libraries like `lodash-es`, which re-exports
+    // a huge number of functions since we can avoid even transforming the files that aren't used.
+    let defer = false;
+    if (dep.isWeak && req.sideEffects === false) {
+      let assets = this.getNodesConnectedTo(depNode);
+      let symbols = invertMap(dep.symbols);
+      invariant(
+        assets[0].type === 'asset' || assets[0].type === 'asset_reference'
+      );
+      let resolvedAsset = assets[0].value;
+      let deps = this.getAncestorDependencies(resolvedAsset);
+      defer = deps.every(
+        d =>
+          !d.symbols.has('*') &&
+          ![...d.symbols.keys()].some(symbol => {
+            let assetSymbol = resolvedAsset.symbols.get(symbol);
+            return assetSymbol != null && symbols.has(assetSymbol);
+          })
+      );
+    }
+
     if (added.nodes.size) {
+      this.deferredNodes.add(requestNode.id);
+    }
+
+    if (!defer && this.deferredNodes.has(requestNode.id)) {
       newRequest = req;
       this.incompleteNodes.set(requestNode.id, requestNode);
+      this.deferredNodes.delete(requestNode.id);
     }
 
     let prunedFiles = getFilesFromGraph(removed);
@@ -211,9 +246,15 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
     let removedFiles = getFilesFromGraph(removed);
 
     for (let assetNode of assetNodes) {
-      let depNodes = assetNode.value
-        .getDependencies()
-        .map(dep => nodeFromDep(dep));
+      let depNodes = assetNode.value.getDependencies().map(dep => {
+        let node = this.getNode(dep.id);
+        if (node && node.type === 'dependency') {
+          node.value.merge(dep);
+          return node;
+        }
+
+        return nodeFromDep(dep);
+      });
       let {removed, added} = this.replaceNodesConnectedTo(assetNode, depNodes);
       removedFiles = removedFiles.concat(getFilesFromGraph(removed));
       newDepNodes = newDepNodes.concat(getDepNodesFromGraph(added));
@@ -281,15 +322,29 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
     return res;
   }
 
-  traverseAssets(
-    visit: GraphTraversalCallback<Asset, AssetGraphNode>,
-    startNode: ?AssetGraphNode
-  ): ?AssetGraphNode {
-    return this.traverse((node, ...args) => {
-      if (node.type === 'asset') {
-        return visit(node.value, ...args);
+  getAncestorDependencies(asset: Asset): Array<IDependency> {
+    let node = this.getNode(asset.id);
+    if (!node) {
+      return [];
+    }
+
+    return this.findAncestors(node, node => node.type === 'dependency').map(
+      node => {
+        invariant(node.type === 'dependency');
+        return node.value;
       }
-    }, startNode);
+    );
+  }
+
+  traverseAssets<TContext>(
+    visit: GraphVisitor<Asset, TContext>,
+    startNode: ?AssetGraphNode
+  ): ?TContext {
+    return this.filteredTraverse(
+      node => (node.type === 'asset' ? node.value : null),
+      visit,
+      startNode
+    );
   }
 
   getTotalSize(asset?: ?Asset): number {
@@ -327,4 +382,37 @@ export default class AssetGraph extends Graph<AssetGraphNode> {
 
     return referenceId;
   }
+
+  resolveSymbol(asset: Asset, symbol: Symbol): SymbolResolution {
+    if (symbol === '*') {
+      return {asset, exportSymbol: '*', symbol: '*'};
+    }
+
+    let identifier = asset.symbols.get(symbol);
+
+    let deps = this.getDependencies(asset).reverse();
+    for (let dep of deps) {
+      // If this is a re-export, find the original module.
+      let symbolLookup = new Map(
+        [...dep.symbols].map(([key, val]) => [val, key])
+      );
+      let depSymbol = symbolLookup.get(identifier);
+      if (depSymbol != null) {
+        let resolved = nullthrows(this.getDependencyResolution(dep));
+        return this.resolveSymbol(resolved, depSymbol);
+      }
+
+      // If this module exports wildcards, resolve the original module.
+      // Default exports are excluded from wildcard exports.
+      if (dep.symbols.get('*') === '*' && symbol !== 'default') {
+        let resolved = nullthrows(this.getDependencyResolution(dep));
+        let result = this.resolveSymbol(resolved, symbol);
+        if (result.symbol != null) {
+          return result;
+        }
+      }
+    }
+
+    return {asset, exportSymbol: symbol, symbol: identifier};
+  }
 }

packages/core/core/src/AssetGraphBuilder.js

@@ -131,9 +131,9 @@ export default class AssetGraphBuilder extends EventEmitter {
   }
 
   async resolve(dep: Dependency, {signal}: BuildOpts) {
-    let resolvedPath;
+    let req;
     try {
-      resolvedPath = await this.resolverRunner.resolve(dep);
+      req = await this.resolverRunner.resolve(dep);
     } catch (err) {
       if (err.code === 'MODULE_NOT_FOUND' && dep.isOptional) {
         return;
@@ -146,9 +146,7 @@ export default class AssetGraphBuilder extends EventEmitter {
       throw new BuildAbortError();
     }
 
-    let req = {filePath: resolvedPath, env: dep.env};
     let {newRequest} = this.graph.resolveDependency(dep, req);
-
     if (newRequest) {
       this.queue.add(() => this.transform(newRequest, {signal}));
       if (this.watcher) this.watcher.watch(newRequest.filePath);

packages/core/core/src/BundleGraph.js

@@ -4,11 +4,11 @@ import type {GraphTraversalCallback} from '@parcel/types';
 import type Asset from './Asset';
 import type {Bundle, BundleGraphNode} from './types';
 
-import Graph from './Graph';
+import Graph, {type GraphOpts} from './Graph';
 
 export default class BundleGraph extends Graph<BundleGraphNode> {
-  constructor() {
-    super();
+  constructor(opts?: GraphOpts<BundleGraphNode>) {
+    super(opts);
     this.setRootNode({
       type: 'root',
       id: 'root',
@@ -46,10 +46,25 @@ export default class BundleGraph extends Graph<BundleGraphNode> {
   traverseBundles<TContext>(
     visit: GraphTraversalCallback<Bundle, TContext>
   ): ?TContext {
-    return this.traverse((node, ...args) => {
-      if (node.type === 'bundle') {
-        return visit(node.value, ...args);
+    return this.filteredTraverse(
+      node => (node.type === 'bundle' ? node.value : null),
+      visit
+    );
+  }
+
+  isAssetReferenced(asset: Asset) {
+    let result = false;
+
+    this.traverseBundles((bundle, context, traversal) => {
+      let referenceNode = bundle.assetGraph.findNode(
+        node => node.type === 'asset_reference' && node.value.id === asset.id
+      );
+      if (referenceNode) {
+        result = true;
+        traversal.stop();
       }
     });
+
+    return result;
   }
 }