From 98464d792093fc30a309fa7d7789775dd68f1403 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 21:20:25 -0700 Subject: [PATCH 1/8] Store graph nodes in array instead of map --- .../bundlers/default/src/DefaultBundler.js | 23 +++++++-------- packages/core/core/src/BundleGraph.js | 9 +++--- packages/core/core/src/RequestTracker.js | 8 +++--- packages/core/core/src/SymbolPropagation.js | 2 +- packages/core/core/src/dumpGraphToGraphViz.js | 4 ++- .../core/src/requests/AssetGraphRequest.js | 2 +- packages/core/core/test/AssetGraph.test.js | 28 +++++++++---------- .../core/core/test/SymbolPropagation.test.js | 14 +++++----- packages/core/graph/src/AdjacencyList.js | 18 ++++++++++++ packages/core/graph/src/Graph.js | 20 ++++++------- packages/core/graph/src/types.js | 2 +- packages/core/graph/test/Graph.test.js | 12 ++++---- packages/dev/query/src/cli.js | 22 +++++++-------- 13 files changed, 92 insertions(+), 72 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 9e018a18226..aede609de1c 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -145,8 +145,8 @@ function decorateLegacyGraph( } = idealGraph; let entryBundleToBundleGroup: Map = new Map(); // Step Create Bundles: Create bundle groups, bundles, and shared bundles and add assets to them - for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - if (idealBundle === 'root') continue; + for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes.entries()) { + if (!idealBundle || idealBundle === 'root') continue; let entryAsset = idealBundle.mainEntryAsset; let bundleGroup; let bundle; @@ -227,8 +227,8 @@ function decorateLegacyGraph( } } // Step Internalization: Internalize dependencies for bundles - for (let [, idealBundle] of idealBundleGraph.nodes) { - if (idealBundle === 'root') continue; + for (let idealBundle of idealBundleGraph.nodes) { + if (!idealBundle || idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); if (idealBundle.internalizedAssets) { for (let internalized of idealBundle.internalizedAssets.values()) { @@ -599,12 +599,13 @@ function createIdealGraph( let assetSet = BitSet.from(assets); // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups - for (let [nodeIdA, a] of bundleGraph.nodes) { + for (let [nodeIdA, a] of bundleGraph.nodes.entries()) { //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes - if (!typeChangeIds.has(nodeIdA) || a === 'root') continue; + if (!a || !typeChangeIds.has(nodeIdA) || a === 'root') continue; let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); - for (let [nodeIdB, b] of bundleGraph.nodes) { + for (let [nodeIdB, b] of bundleGraph.nodes.entries()) { if ( + b && a !== 'root' && b !== 'root' && a !== b && @@ -830,8 +831,8 @@ function createIdealGraph( // the bundle is synchronously available elsewhere. // We can query sync assets available via reachableRoots. If the parent has // the bundleRoot by reachableRoots AND ancestorAssets, internalize it. - for (let [id, bundleRoot] of bundleRootGraph.nodes) { - if (bundleRoot === 'root') continue; + for (let [id, bundleRoot] of bundleRootGraph.nodes.entries()) { + if (!bundleRoot || bundleRoot === 'root') continue; let parentRoots = bundleRootGraph .getNodeIdsConnectedTo(id, ALL_EDGE_TYPES) .map(id => nullthrows(bundleRootGraph.getNode(id))); @@ -1051,8 +1052,8 @@ function createIdealGraph( // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into // their source bundles, and remove the bundle. // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained - for (let [bundleNodeId, bundle] of bundleGraph.nodes) { - if (bundle === 'root') continue; + for (let [bundleNodeId, bundle] of bundleGraph.nodes.entries()) { + if (!bundle || bundle === 'root') continue; if ( bundle.sourceBundles.size > 0 && bundle.mainEntryAsset == null && diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index c1a32bd002a..a5af4ecf59e 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -174,8 +174,8 @@ export default class BundleGraph { : null; invariant(assetGraphRootNode != null && assetGraphRootNode.type === 'root'); - for (let [nodeId, node] of assetGraph.nodes) { - if (node.type === 'asset') { + for (let [nodeId, node] of assetGraph.nodes.entries()) { + if (node != null && node.type === 'asset') { let {id: assetId} = node.value; // Generate a new, short public id for this asset to use. // If one already exists, use it. @@ -187,7 +187,7 @@ export default class BundleGraph { publicIdByAssetId.set(assetId, publicId); assetPublicIds.add(publicId); } - } else if (node.type === 'asset_group') { + } else if (node != null && node.type === 'asset_group') { assetGroupIds.set(nodeId, assetGraph.getNodeIdsConnectedFrom(nodeId)); } } @@ -2011,7 +2011,8 @@ export default class BundleGraph { merge(other: BundleGraph) { let otherGraphIdToThisNodeId = new Map(); - for (let [otherNodeId, otherNode] of other._graph.nodes) { + for (let [otherNodeId, otherNode] of other._graph.nodes.entries()) { + if (!otherNode) continue; if (this._graph.hasContentKey(otherNode.id)) { let existingNodeId = this._graph.getNodeIdByContentKey(otherNode.id); otherGraphIdToThisNodeId.set(otherNodeId, existingNodeId); diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index 38885aa829a..c318546ae00 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -724,8 +724,8 @@ export class RequestGraph extends ContentGraph< // this means the project root was moved and we need to // re-run all requests. if (type === 'create' && filePath === '') { - for (let [id, node] of this.nodes) { - if (node.type === 'request') { + for (let [id, node] of this.nodes.entries()) { + if (node?.type === 'request') { this.invalidNodeIds.add(id); } } @@ -1087,8 +1087,8 @@ export default class RequestTracker { } let promises = []; - for (let [, node] of this.graph.nodes) { - if (node.type !== 'request') { + for (let node of this.graph.nodes) { + if (!node || node.type !== 'request') { continue; } diff --git a/packages/core/core/src/SymbolPropagation.js b/packages/core/core/src/SymbolPropagation.js index 04f0084b59d..4fcd91ef650 100644 --- a/packages/core/core/src/SymbolPropagation.js +++ b/packages/core/core/src/SymbolPropagation.js @@ -629,7 +629,7 @@ function propagateSymbolsUp( let runFullPass = // If there are n nodes in the graph, then the asset count is approximately // n/6 (for every asset, there are ~4 dependencies and ~1 asset_group). - assetGraph.nodes.size * (1 / 6) * 0.5 < + assetGraph.nodes.length * (1 / 6) * 0.5 < changedDepsUsedSymbolsUpDirtyDownAssets.size; let dirtyDeps; diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index 9b7bcfffe4b..f07983cf9dd 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -60,7 +60,9 @@ export default async function dumpGraphToGraphViz( const graphviz = require('graphviz'); const tempy = require('tempy'); let g = graphviz.digraph('G'); - for (let [id, node] of graph.nodes) { + // $FlowFixMe + for (let [id, node] of graph.nodes.entries()) { + if (node == null) continue; let n = g.addNode(nodeId(id)); // $FlowFixMe default is fine. Not every type needs to be in the map. n.set('color', COLORS[node.type || 'default']); diff --git a/packages/core/core/src/requests/AssetGraphRequest.js b/packages/core/core/src/requests/AssetGraphRequest.js index 29b275d3a15..0895418ac59 100644 --- a/packages/core/core/src/requests/AssetGraphRequest.js +++ b/packages/core/core/src/requests/AssetGraphRequest.js @@ -242,7 +242,7 @@ export class AssetGraphBuilder { throw errors[0]; } - if (this.assetGraph.nodes.size > 1) { + if (this.assetGraph.nodes.length > 1) { await dumpGraphToGraphViz( this.assetGraph, 'AssetGraph_' + this.name + '_before_prop', diff --git a/packages/core/core/test/AssetGraph.test.js b/packages/core/core/test/AssetGraph.test.js index 5f5fc249f1a..6737acfa912 100644 --- a/packages/core/core/test/AssetGraph.test.js +++ b/packages/core/core/test/AssetGraph.test.js @@ -260,7 +260,7 @@ describe('AssetGraph', () => { nodeFromAssetGroup(req).id, ); let dependencyNodeId = graph.getNodeIdByContentKey(dep.id); - assert(graph.nodes.has(assetGroupNodeId)); + assert(graph.hasNode(assetGroupNodeId)); assert(graph.hasEdge(dependencyNodeId, assetGroupNodeId)); let req2 = { @@ -272,13 +272,13 @@ describe('AssetGraph', () => { let assetGroupNodeId2 = graph.getNodeIdByContentKey( nodeFromAssetGroup(req2).id, ); - assert(!graph.nodes.has(assetGroupNodeId)); - assert(graph.nodes.has(assetGroupNodeId2)); + assert(!graph.hasNode(assetGroupNodeId)); + assert(graph.hasNode(assetGroupNodeId2)); assert(graph.hasEdge(dependencyNodeId, assetGroupNodeId2)); assert(!graph.hasEdge(dependencyNodeId, assetGroupNodeId)); graph.resolveDependency(dep, req2, '5'); - assert(graph.nodes.has(assetGroupNodeId2)); + assert(graph.hasNode(assetGroupNodeId2)); assert(graph.hasEdge(dependencyNodeId, assetGroupNodeId2)); }); @@ -389,11 +389,11 @@ describe('AssetGraph', () => { [...assets[1].dependencies.values()][0].id, ); - assert(graph.nodes.has(nodeId1)); - assert(graph.nodes.has(nodeId2)); - assert(graph.nodes.has(nodeId3)); - assert(graph.nodes.has(dependencyNodeId1)); - assert(graph.nodes.has(dependencyNodeId2)); + assert(graph.hasNode(nodeId1)); + assert(graph.hasNode(nodeId2)); + assert(graph.hasNode(nodeId3)); + assert(graph.hasNode(dependencyNodeId1)); + assert(graph.hasNode(dependencyNodeId2)); assert(graph.hasEdge(assetGroupNode, nodeId1)); assert(graph.hasEdge(assetGroupNode, nodeId2)); assert(graph.hasEdge(assetGroupNode, nodeId3)); @@ -435,11 +435,11 @@ describe('AssetGraph', () => { graph.resolveAssetGroup(req, assets2, '5'); - assert(graph.nodes.has(nodeId1)); - assert(graph.nodes.has(nodeId2)); - assert(!graph.nodes.has(nodeId3)); - assert(graph.nodes.has(dependencyNodeId1)); - assert(!graph.nodes.has(dependencyNodeId2)); + assert(graph.hasNode(nodeId1)); + assert(graph.hasNode(nodeId2)); + assert(!graph.hasNode(nodeId3)); + assert(graph.hasNode(dependencyNodeId1)); + assert(!graph.hasNode(dependencyNodeId2)); assert(graph.hasEdge(assetGroupNode, nodeId1)); assert(graph.hasEdge(assetGroupNode, nodeId2)); assert(!graph.hasEdge(assetGroupNode, nodeId3)); diff --git a/packages/core/core/test/SymbolPropagation.test.js b/packages/core/core/test/SymbolPropagation.test.js index 8e4a597267f..09a01374341 100644 --- a/packages/core/core/test/SymbolPropagation.test.js +++ b/packages/core/core/test/SymbolPropagation.test.js @@ -197,7 +197,7 @@ function assertUsedSymbols( if (isLibrary) { let entryDep = nullthrows( [...graph.nodes.values()].find( - n => n.type === 'dependency' && n.value.sourceAssetId == null, + n => n?.type === 'dependency' && n.value.sourceAssetId == null, ), ); invariant(entryDep.type === 'dependency'); @@ -240,12 +240,12 @@ function assertUsedSymbols( } } - for (let [nodeId, node] of graph.nodes) { - if (node.type === 'asset') { + for (let [nodeId, node] of graph.nodes.entries()) { + if (node?.type === 'asset') { let filePath = fromProjectPathUnix(node.value.filePath); let expected = new Set(nullthrows(expectedAsset.get(filePath))); assertSetEqual(node.usedSymbols, expected, filePath); - } else if (node.type === 'dependency' && node.value.sourcePath != null) { + } else if (node?.type === 'dependency' && node.value.sourcePath != null) { let resolutionId = graph.getNodeIdsConnectedFrom(nodeId)[0]; let resolution = nullthrows(graph.getNode(resolutionId)); invariant(resolution.type === 'asset_group'); @@ -373,14 +373,14 @@ function changeDependency( ): Iterable<[ContentKey, Asset]> { let sourceAssetNode = nullthrowsAssetNode( [...graph.nodes.values()].find( - n => n.type === 'asset' && n.value.filePath === from, + n => n?.type === 'asset' && n.value.filePath === from, ), ); sourceAssetNode.usedSymbolsDownDirty = true; let depNode = nullthrowsDependencyNode( [...graph.nodes.values()].find( n => - n.type === 'dependency' && + n?.type === 'dependency' && n.value.sourcePath === from && n.value.specifier === to, ), @@ -396,7 +396,7 @@ function changeAsset( ): Iterable<[ContentKey, Asset]> { let node = nullthrowsAssetNode( [...graph.nodes.values()].find( - n => n.type === 'asset' && n.value.filePath === asset, + n => n?.type === 'asset' && n.value.filePath === asset, ), ); node.usedSymbolsUpDirty = true; diff --git a/packages/core/graph/src/AdjacencyList.js b/packages/core/graph/src/AdjacencyList.js index 73a18ae7d5f..d4adb0657ce 100644 --- a/packages/core/graph/src/AdjacencyList.js +++ b/packages/core/graph/src/AdjacencyList.js @@ -467,6 +467,24 @@ export default class AdjacencyList { return nodes; } + forEachNodeIdConnectedFromReverse( + from: NodeId, + fn: (nodeId: NodeId) => boolean, + ) { + let node = this.#nodes.head(from); + while (node !== null) { + let edge = this.#nodes.lastOut(node); + while (edge !== null) { + let to = this.#edges.to(edge); + if (fn(to)) { + return; + } + edge = this.#edges.prevOut(edge); + } + node = this.#nodes.next(node); + } + } + /** * Get the list of nodes connected to this node. */ diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index 8fb1a3f4024..b407d4b30ed 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -5,18 +5,17 @@ import AdjacencyList, {type SerializedAdjacencyList} from './AdjacencyList'; import type {Edge, NodeId} from './types'; import type {TraversalActions, GraphVisitor} from '@parcel/types'; -import assert from 'assert'; import nullthrows from 'nullthrows'; export type NullEdgeType = 1; export type GraphOpts = {| - nodes?: Map, + nodes?: Array, adjacencyList?: SerializedAdjacencyList, rootNodeId?: ?NodeId, |}; export type SerializedGraph = {| - nodes: Map, + nodes: Array, adjacencyList: SerializedAdjacencyList, rootNodeId: ?NodeId, |}; @@ -25,12 +24,12 @@ export type AllEdgeTypes = -1; export const ALL_EDGE_TYPES: AllEdgeTypes = -1; export default class Graph { - nodes: Map; + nodes: Array; adjacencyList: AdjacencyList; rootNodeId: ?NodeId; constructor(opts: ?GraphOpts) { - this.nodes = opts?.nodes || new Map(); + this.nodes = opts?.nodes || []; this.setRootNodeId(opts?.rootNodeId); let adjacencyList = opts?.adjacencyList; @@ -69,16 +68,16 @@ export default class Graph { addNode(node: TNode): NodeId { let id = this.adjacencyList.addNode(); - this.nodes.set(id, node); + this.nodes.push(node); return id; } hasNode(id: NodeId): boolean { - return this.nodes.has(id); + return this.nodes[id] != null; } getNode(id: NodeId): ?TNode { - return this.nodes.get(id); + return this.nodes[id]; } addEdge( @@ -156,8 +155,7 @@ export default class Graph { this._removeEdge(nodeId, to, type); } - let wasRemoved = this.nodes.delete(nodeId); - assert(wasRemoved); + this.nodes[nodeId] = null; } removeEdges(nodeId: NodeId, type: TEdgeType | NullEdgeType = 1) { @@ -237,7 +235,7 @@ export default class Graph { updateNode(nodeId: NodeId, node: TNode): void { this._assertHasNodeId(nodeId); - this.nodes.set(nodeId, node); + this.nodes[nodeId] = node; } // Update a node's downstream nodes making sure to prune any orphaned branches diff --git a/packages/core/graph/src/types.js b/packages/core/graph/src/types.js index 355f02d08a8..ff2607ab1fb 100644 --- a/packages/core/graph/src/types.js +++ b/packages/core/graph/src/types.js @@ -1,7 +1,7 @@ // @flow strict-local // forcing NodeId to be opaque as it should only be created once -export opaque type NodeId = number; +export type NodeId = number; export function toNodeId(x: number): NodeId { return x; } diff --git a/packages/core/graph/test/Graph.test.js b/packages/core/graph/test/Graph.test.js index 6cf42d30ff1..7a045dd9b97 100644 --- a/packages/core/graph/test/Graph.test.js +++ b/packages/core/graph/test/Graph.test.js @@ -17,7 +17,7 @@ describe('Graph', () => { let graph = new Graph(); let node = {}; let id = graph.addNode(node); - assert.equal(graph.nodes.get(id), node); + assert.equal(graph.getNode(id), node); }); it('errors when traversing a graph with no root', () => { @@ -117,10 +117,10 @@ describe('Graph', () => { graph.addEdge(nodeB, nodeD); graph.removeEdge(nodeA, nodeB); - assert(graph.nodes.has(nodeA)); - assert(graph.nodes.has(nodeD)); - assert(!graph.nodes.has(nodeB)); - assert(!graph.nodes.has(nodeC)); + assert(graph.hasNode(nodeA)); + assert(graph.hasNode(nodeD)); + assert(!graph.hasNode(nodeB)); + assert(!graph.hasNode(nodeC)); assert.deepEqual( [...graph.getAllEdges()], [{from: nodeA, to: nodeD, type: 1}], @@ -337,7 +337,7 @@ describe('Graph', () => { graph.removeNode(node1); - assert.strictEqual(graph.nodes.size, 1); + assert.strictEqual(graph.nodes.length, 1); assert.deepStrictEqual(Array.from(graph.getAllEdges()), []); }); }); diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index 8a6793a7739..13e7b546ff2 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -87,7 +87,7 @@ export function run(input: string[]) { let assetRegex = new RegExp(v); for (let node of assetGraph.nodes.values()) { if ( - node.type === 'asset' && + node?.type === 'asset' && assetRegex.test(fromProjectPathRelative(node.value.filePath)) ) { id = node.id; @@ -129,7 +129,7 @@ export function run(input: string[]) { let assetRegex = new RegExp(v); for (let node of assetGraph.nodes.values()) { if ( - node.type === 'asset' && + node?.type === 'asset' && assetRegex.test(fromProjectPathRelative(node.value.filePath)) ) { return node; @@ -162,7 +162,7 @@ export function run(input: string[]) { // Search against the id used by the JSTransformer and ScopeHoistingPackager, // not the final asset id, as it may have changed with further transformation. for (let node of assetGraph.nodes.values()) { - if (node.type === 'asset' && node.value.meta.id === assetId) { + if (node?.type === 'asset' && node.value.meta.id === assetId) { asset = node; break; } @@ -172,7 +172,7 @@ export function run(input: string[]) { // search for the local name in asset used symbols. if (asset == null) { outer: for (let node of assetGraph.nodes.values()) { - if (node.type === 'asset' && node.value.symbols) { + if (node?.type === 'asset' && node.value.symbols) { for (let symbol of node.value.symbols.values()) { if (symbol.local === local) { asset = node; @@ -613,8 +613,8 @@ export function run(input: string[]) { asset_group: 0, }; - for (let [, n] of assetGraph.nodes) { - if (n.type in ag) { + for (let n of assetGraph.nodes) { + if (n && n.type in ag) { // $FlowFixMe ag[n.type]++; } @@ -640,10 +640,10 @@ export function run(input: string[]) { const entries = new Set(); - for (let [, n] of bundleGraph._graph.nodes) { - if (n.type === 'bundle_group') { + for (let n of bundleGraph._graph.nodes) { + if (n?.type === 'bundle_group') { bg.bundle_group++; - } else if (n.type === 'bundle') { + } else if (n?.type === 'bundle') { bg.bundle++; // $FlowFixMe @@ -669,7 +669,7 @@ export function run(input: string[]) { b_type.sync++; } } - } else if (n.type === 'asset') { + } else if (n?.type === 'asset') { if ( // $FlowFixMe fromProjectPathRelative(n.value.filePath).includes('node_modules') @@ -678,7 +678,7 @@ export function run(input: string[]) { } else { bg.asset_source++; } - } else if (n.type === 'dependency') { + } else if (n?.type === 'dependency') { bg.dependency++; } } From d307163e8aefe0060ed72872b8e71e0af1b1fa74 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 21:23:55 -0700 Subject: [PATCH 2/8] Use function to get public dependencies instead of constructor --- packages/core/core/src/public/Asset.js | 10 ++++++++-- packages/core/core/src/public/Bundle.js | 7 +++++-- packages/core/core/src/public/BundleGraph.js | 9 ++++++--- packages/core/core/src/public/Dependency.js | 17 ++++++++++++----- packages/core/core/src/requests/PathRequest.js | 4 ++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/packages/core/core/src/public/Asset.js b/packages/core/core/src/public/Asset.js index 1c706577889..506144d65fe 100644 --- a/packages/core/core/src/public/Asset.js +++ b/packages/core/core/src/public/Asset.js @@ -25,7 +25,7 @@ import type {Asset as AssetValue, ParcelOptions} from '../types'; import nullthrows from 'nullthrows'; import Environment from './Environment'; -import Dependency from './Dependency'; +import {getPublicDependency} from './Dependency'; import {AssetSymbols, MutableAssetSymbols} from './Symbols'; import UncommittedAsset from '../UncommittedAsset'; import CommittedAsset from '../CommittedAsset'; @@ -162,7 +162,7 @@ class BaseAsset { getDependencies(): $ReadOnlyArray { return this.#asset .getDependencies() - .map(dep => new Dependency(dep, this.#asset.options)); + .map(dep => getPublicDependency(dep, this.#asset.options)); } getCode(): Promise { @@ -192,6 +192,7 @@ class BaseAsset { export class Asset extends BaseAsset implements IAsset { #asset /*: CommittedAsset | UncommittedAsset */; + #env /*: ?Environment */; constructor(asset: CommittedAsset | UncommittedAsset): Asset { let assetValueToAsset = asset.value.committed @@ -208,6 +209,11 @@ export class Asset extends BaseAsset implements IAsset { return this; } + get env(): IEnvironment { + this.#env ??= new Environment(this.#asset.value.env, this.#asset.options); + return this.#env; + } + get stats(): Stats { return this.#asset.value.stats; } diff --git a/packages/core/core/src/public/Bundle.js b/packages/core/core/src/public/Bundle.js index fb95d1a5359..c138b67c737 100644 --- a/packages/core/core/src/public/Bundle.js +++ b/packages/core/core/src/public/Bundle.js @@ -27,7 +27,10 @@ import {DefaultWeakMap} from '@parcel/utils'; import {assetToAssetValue, assetFromValue} from './Asset'; import {mapVisitor} from '@parcel/graph'; import Environment from './Environment'; -import Dependency, {dependencyToInternalDependency} from './Dependency'; +import { + dependencyToInternalDependency, + getPublicDependency, +} from './Dependency'; import Target from './Target'; import {BundleBehaviorNames} from '../types'; import {fromProjectPath} from '../projectPath'; @@ -179,7 +182,7 @@ export class Bundle implements IBundle { } else if (node.type === 'dependency') { return { type: 'dependency', - value: new Dependency(node.value, this.#options), + value: getPublicDependency(node.value, this.#options), }; } }, visit), diff --git a/packages/core/core/src/public/BundleGraph.js b/packages/core/core/src/public/BundleGraph.js index b42a7b68fed..1044a0ad9cd 100644 --- a/packages/core/core/src/public/BundleGraph.js +++ b/packages/core/core/src/public/BundleGraph.js @@ -23,7 +23,10 @@ import nullthrows from 'nullthrows'; import {mapVisitor} from '@parcel/graph'; import {assetFromValue, assetToAssetValue, Asset} from './Asset'; import {bundleToInternalBundle} from './Bundle'; -import Dependency, {dependencyToInternalDependency} from './Dependency'; +import Dependency, { + dependencyToInternalDependency, + getPublicDependency, +} from './Dependency'; import {targetToInternalTarget} from './Target'; import {fromInternalSourceLocation} from '../utils'; import BundleGroup, {bundleGroupToInternalBundleGroup} from './BundleGroup'; @@ -90,7 +93,7 @@ export default class BundleGraph getIncomingDependencies(asset: IAsset): Array { return this.#graph .getIncomingDependencies(assetToAssetValue(asset)) - .map(dep => new Dependency(dep, this.#options)); + .map(dep => getPublicDependency(dep, this.#options)); } getAssetWithDependency(dep: IDependency): ?IAsset { @@ -158,7 +161,7 @@ export default class BundleGraph getDependencies(asset: IAsset): Array { return this.#graph .getDependencies(assetToAssetValue(asset)) - .map(dep => new Dependency(dep, this.#options)); + .map(dep => getPublicDependency(dep, this.#options)); } isAssetReachableFromBundle(asset: IAsset, bundle: IBundle): boolean { diff --git a/packages/core/core/src/public/Dependency.js b/packages/core/core/src/public/Dependency.js index d145768f5fb..bc060fbece2 100644 --- a/packages/core/core/src/public/Dependency.js +++ b/packages/core/core/src/public/Dependency.js @@ -42,16 +42,23 @@ export function dependencyToInternalDependency( return nullthrows(_dependencyToInternalDependency.get(dependency)); } +export function getPublicDependency( + dep: InternalDependency, + options: ParcelOptions, +): Dependency { + let existing = internalDependencyToDependency.get(dep); + if (existing != null) { + return existing; + } + + return new Dependency(dep, options); +} + export default class Dependency implements IDependency { #dep /*: InternalDependency */; #options /*: ParcelOptions */; constructor(dep: InternalDependency, options: ParcelOptions): Dependency { - let existing = internalDependencyToDependency.get(dep); - if (existing != null) { - return existing; - } - this.#dep = dep; this.#options = options; _dependencyToInternalDependency.set(this, dep); diff --git a/packages/core/core/src/requests/PathRequest.js b/packages/core/core/src/requests/PathRequest.js index fb774d53d06..782355b073c 100644 --- a/packages/core/core/src/requests/PathRequest.js +++ b/packages/core/core/src/requests/PathRequest.js @@ -26,7 +26,7 @@ import nullthrows from 'nullthrows'; import path from 'path'; import {normalizePath} from '@parcel/utils'; import {report} from '../ReporterRunner'; -import PublicDependency from '../public/Dependency'; +import {getPublicDependency} from '../public/Dependency'; import PluginOptions from '../public/PluginOptions'; import ParcelConfig from '../ParcelConfig'; import createParcelConfigRequest, { @@ -240,7 +240,7 @@ export class ResolverRunner { } async resolve(dependency: Dependency): Promise { - let dep = new PublicDependency(dependency, this.options); + let dep = getPublicDependency(dependency, this.options); report({ type: 'buildProgress', phase: 'resolving', From 670084929cd5d52c792f6e68eaa4fcefd82b216e Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 21:25:54 -0700 Subject: [PATCH 3/8] Faster BitSet --- packages/core/utils/src/BitSet.js | 150 +++++++++++++++++++++--------- packages/core/utils/src/index.js | 2 +- 2 files changed, 108 insertions(+), 44 deletions(-) diff --git a/packages/core/utils/src/BitSet.js b/packages/core/utils/src/BitSet.js index 39db524338c..71e2bce064f 100644 --- a/packages/core/utils/src/BitSet.js +++ b/packages/core/utils/src/BitSet.js @@ -1,23 +1,9 @@ // @flow strict-local import nullthrows from 'nullthrows'; -// As our current version of flow doesn't support BigInt's, these values/types -// have been hoisted to keep the flow errors to a minimum. This can be removed -// if we upgrade to a flow version that supports BigInt's -// $FlowFixMe -type TmpBigInt = bigint; -// $FlowFixMe -const BIGINT_ZERO = 0n; -// $FlowFixMe -const BIGINT_ONE = 1n; -// $FlowFixMe -let numberToBigInt = (v: number): TmpBigInt => BigInt(v); - -let bitUnion = (a: TmpBigInt, b: TmpBigInt): TmpBigInt => a | b; - export class BitSet { - _value: TmpBigInt; - _lookup: Map; + _value: RawBitSet; + _lookup: Map; _items: Array; constructor({ @@ -27,14 +13,14 @@ export class BitSet { }: {| items: Array, lookup: Map, - initial?: BitSet | TmpBigInt, + initial?: BitSet | RawBitSet, |}) { if (initial instanceof BitSet) { this._value = initial?._value; } else if (initial) { this._value = initial; } else { - this._value = BIGINT_ZERO; + this._value = new RawBitSet(items.length); } this._items = items; @@ -42,17 +28,19 @@ export class BitSet { } static from(items: Array): BitSet { - let lookup: Map = new Map(); + let lookup: Map = new Map(); for (let i = 0; i < items.length; i++) { - lookup.set(items[i], numberToBigInt(i)); + lookup.set(items[i], i); } return new BitSet({items, lookup}); } static union(a: BitSet, b: BitSet): BitSet { + let value = a._value.clone(); + value.union(b._value); return new BitSet({ - initial: bitUnion(a._value, b._value), + initial: value, lookup: a._lookup, items: a._items, }); @@ -63,27 +51,31 @@ export class BitSet { } add(item: Item) { - this._value |= BIGINT_ONE << this.#getIndex(item); + this._value.add(this.#getIndex(item)); } delete(item: Item) { - this._value &= ~(BIGINT_ONE << this.#getIndex(item)); + this._value.delete(this.#getIndex(item)); } has(item: Item): boolean { - return Boolean(this._value & (BIGINT_ONE << this.#getIndex(item))); + return this._value.has(this.#getIndex(item)); } intersect(v: BitSet) { - this._value = this._value & v._value; + this._value.intersect(v._value); } union(v: BitSet) { - this._value = bitUnion(this._value, v._value); + this._value.union(v._value); + } + + unionRaw(v: RawBitSet) { + this._value.union(v); } clear() { - this._value = BIGINT_ZERO; + this._value.clear(); } cloneEmpty(): BitSet { @@ -97,30 +89,102 @@ export class BitSet { return new BitSet({ lookup: this._lookup, items: this._items, - initial: this._value, + initial: this._value.clone(), }); } values(): Array { let values = []; - let tmpValue = this._value; - let i; - - // This implementation is optimized for BitSets that contain a very small percentage - // of items compared to the total number of potential items. This makes sense for - // our bundler use-cases where Sets often contain <1% coverage of the total item count. - // In cases where Sets contain a larger percentage of the total items, a regular looping - // strategy would be more performant. - while (tmpValue > BIGINT_ZERO) { - // Get last set bit - i = tmpValue.toString(2).length - 1; - + this._value.forEach(i => { values.push(this._items[i]); + }); + + return values; + } +} + +// Small wasm program that exposes the `ctz` instruction. +// https://developer.mozilla.org/en-US/docs/WebAssembly/Reference/Numeric/Count_trailing_zeros +const wasmBuf = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, 0x01, + 0x7f, 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0d, 0x01, 0x09, 0x74, 0x72, + 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x30, 0x00, 0x00, 0x0a, 0x07, 0x01, 0x05, + 0x00, 0x20, 0x00, 0x68, 0x0b, 0x00, 0x0f, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x02, + 0x08, 0x01, 0x00, 0x01, 0x00, 0x03, 0x6e, 0x75, 0x6d, +]); + +// eslint-disable-next-line +const {trailing0} = new WebAssembly.Instance(new WebAssembly.Module(wasmBuf)) + .exports; + +export class RawBitSet { + bits: Uint32Array; + + constructor(maxBits: number) { + this.bits = new Uint32Array(Math.ceil(maxBits / 32)); + } + + clone(): RawBitSet { + let res = new RawBitSet(this.capacity); + res.bits.set(this.bits); + return res; + } + + get capacity(): number { + return this.bits.length * 32; + } + + add(bit: number) { + let i = bit >>> 5; + let b = bit & 31; + this.bits[i] |= 1 << b; + } - // Unset last set bit - tmpValue &= ~(BIGINT_ONE << numberToBigInt(i)); + delete(bit: number) { + let i = bit >>> 5; + let b = bit & 31; + this.bits[i] &= ~(1 << b); + } + + has(bit: number): boolean { + let i = bit >>> 5; + let b = bit & 31; + return Boolean(this.bits[i] & (1 << b)); + } + + clear() { + this.bits.fill(0); + } + + intersect(other: RawBitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] &= other.bits[i]; + } + } + + union(other: RawBitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] |= other.bits[i]; } + } - return values; + remove(other: RawBitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] &= ~other.bits[i]; + } + } + + forEach(fn: (bit: number) => void) { + // https://lemire.me/blog/2018/02/21/iterating-over-set-bits-quickly/ + let bits = this.bits; + for (let k = 0; k < bits.length; k++) { + let v = bits[k]; + while (v !== 0) { + let t = (v & -v) >>> 0; + // $FlowFixMe + fn((k << 5) + trailing0(v)); + v ^= t; + } + } } } diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index e01f7d41f6d..179a4d226c4 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -85,5 +85,5 @@ export { loadSourceMap, remapSourceLocation, } from './sourcemap'; -export {BitSet} from './BitSet'; +export {BitSet, RawBitSet} from './BitSet'; export {default as stripAnsi} from 'strip-ansi'; From 461ed1e2da438b3ba73d44ce28dab2dc8828cc80 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 21:26:59 -0700 Subject: [PATCH 4/8] Fast path for depth first search --- packages/core/graph/package.json | 1 + packages/core/graph/src/Graph.js | 109 ++++++++++++++++++++++++++++--- packages/core/types/index.js | 1 + 3 files changed, 103 insertions(+), 8 deletions(-) diff --git a/packages/core/graph/package.json b/packages/core/graph/package.json index 9423ebc33ae..797258db06f 100644 --- a/packages/core/graph/package.json +++ b/packages/core/graph/package.json @@ -20,6 +20,7 @@ "node": ">= 12.0.0" }, "dependencies": { + "@parcel/utils": "^2.9.3", "nullthrows": "^1.1.1" } } diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index b407d4b30ed..1f3c991a833 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -3,7 +3,12 @@ import {fromNodeId} from './types'; import AdjacencyList, {type SerializedAdjacencyList} from './AdjacencyList'; import type {Edge, NodeId} from './types'; -import type {TraversalActions, GraphVisitor} from '@parcel/types'; +import type { + TraversalActions, + GraphVisitor, + GraphTraversalCallback, +} from '@parcel/types'; +import {RawBitSet} from '@parcel/utils'; import nullthrows from 'nullthrows'; @@ -27,6 +32,7 @@ export default class Graph { nodes: Array; adjacencyList: AdjacencyList; rootNodeId: ?NodeId; + _visited: ?RawBitSet; constructor(opts: ?GraphOpts) { this.nodes = opts?.nodes || []; @@ -275,11 +281,20 @@ export default class Graph { | Array | AllEdgeTypes = 1, ): ?TContext { - return this.dfs({ - visit, - startNodeId, - getChildren: nodeId => this.getNodeIdsConnectedFrom(nodeId, type), - }); + let enter = typeof visit === 'function' ? visit : visit.enter; + if ( + type === ALL_EDGE_TYPES && + enter && + (typeof visit === 'function' || !visit.exit) + ) { + return this.dfsFast(enter, startNodeId); + } else { + return this.dfs({ + visit, + startNodeId, + getChildren: nodeId => this.getNodeIdsConnectedFrom(nodeId, type), + }); + } } filteredTraverse( @@ -307,6 +322,72 @@ export default class Graph { }); } + dfsFast( + visit: GraphTraversalCallback, + startNodeId: ?NodeId, + ): ?TContext { + let traversalStartNode = nullthrows( + startNodeId ?? this.rootNodeId, + 'A start node is required to traverse', + ); + this._assertHasNodeId(traversalStartNode); + + let visited; + if (!this._visited || this._visited.capacity < this.nodes.length) { + this._visited = new RawBitSet(this.nodes.length); + visited = this._visited; + } else { + visited = this._visited; + visited.clear(); + } + // Take shared instance to avoid re-entrancy issues. + this._visited = null; + + let stopped = false; + let skipped = false; + let actions: TraversalActions = { + skipChildren() { + skipped = true; + }, + stop() { + stopped = true; + }, + }; + + let queue = [{nodeId: traversalStartNode, context: null}]; + while (queue.length !== 0) { + let {nodeId, context} = queue.pop(); + if (!this.hasNode(nodeId) || visited.has(nodeId)) continue; + visited.add(nodeId); + + skipped = false; + let newContext = visit(nodeId, context, actions); + if (typeof newContext !== 'undefined') { + // $FlowFixMe[reassign-const] + context = newContext; + } + + if (skipped) { + continue; + } + + if (stopped) { + this._visited = visited; + return context; + } + + this.adjacencyList.forEachNodeIdConnectedFromReverse(nodeId, child => { + if (!visited.has(child)) { + queue.push({nodeId: child, context}); + } + return false; + }); + } + + this._visited = visited; + return null; + } + dfs({ visit, startNodeId, @@ -322,7 +403,17 @@ export default class Graph { ); this._assertHasNodeId(traversalStartNode); - let visited = new Set(); + let visited; + if (!this._visited || this._visited.capacity < this.nodes.length) { + this._visited = new RawBitSet(this.nodes.length); + visited = this._visited; + } else { + visited = this._visited; + visited.clear(); + } + // Take shared instance to avoid re-entrancy issues. + this._visited = null; + let stopped = false; let skipped = false; let actions: TraversalActions = { @@ -390,7 +481,9 @@ export default class Graph { } }; - return walk(traversalStartNode); + let result = walk(traversalStartNode); + this._visited = visited; + return result; } bfs(visit: (nodeId: NodeId) => ?boolean): ?NodeId { diff --git a/packages/core/types/index.js b/packages/core/types/index.js index 31728405c28..78c7dca5a25 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1450,6 +1450,7 @@ export interface BundleGraph { traverse( visit: GraphVisitor, startAsset: ?Asset, + options?: {|skipUnusedDependencies?: boolean|}, ): ?TContext; /** Traverses all bundles in the bundle graph, including inline bundles, in depth first order. */ traverseBundles( From 01e8fbe67c91f55ca5203a1ddaa7d0a2048eff59 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 21:28:51 -0700 Subject: [PATCH 5/8] Faster data structures in default bundler --- .../bundlers/default/src/DefaultBundler.js | 618 +++++++++--------- packages/core/core/src/public/BundleGraph.js | 29 +- 2 files changed, 322 insertions(+), 325 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index aede609de1c..4b90178c9a1 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -20,7 +20,13 @@ import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; import logger from '@parcel/logger'; -import {setEqual, validateSchema, DefaultMap, BitSet} from '@parcel/utils'; +import { + setEqual, + validateSchema, + DefaultMap, + BitSet, + RawBitSet, +} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -333,9 +339,6 @@ function createIdealGraph( let bundleGroupBundleIds: Set = new Set(); - // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); - let rootNodeId = nullthrows(bundleRootGraph.addNode('root')); let bundleGraphRootNodeId = nullthrows(bundleGraph.addNode('root')); bundleRootGraph.setRootNodeId(rootNodeId); @@ -371,6 +374,7 @@ function createIdealGraph( } let assets = []; + let assetToIndex = new Map(); let typeChangeIds = new Set(); /** @@ -378,191 +382,85 @@ function createIdealGraph( * for asset type changes, parallel, inline, and async or lazy dependencies, * adding only that asset to each bundle, not its entire subgraph. */ - assetGraph.traverse({ - enter(node, context, actions) { - if (node.type === 'asset') { - if ( - context?.type === 'dependency' && - context?.value.isEntry && - !entries.has(node.value) - ) { - // Skip whole subtrees of other targets by skipping those entries - actions.skipChildren(); - return node; - } - assets.push(node.value); - - let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { - // Push to the stack (only) when a new bundle is created - stack.push([node.value, bundleIdTuple[0]]); - } else if (bundleIdTuple) { - // Otherwise, push on the last bundle that marks the start of a BundleGroup - stack.push([node.value, stack[stack.length - 1][1]]); - } - } else if (node.type === 'dependency') { - if (context == null) { - return node; - } - let dependency = node.value; - - if (assetGraph.isDependencySkipped(dependency)) { - actions.skipChildren(); - return node; - } - - invariant(context?.type === 'asset'); - let parentAsset = context.value; - - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return node; - } - - for (let childAsset of assets) { + assetGraph.traverse( + { + enter(node, context, actions) { + if (node.type === 'asset') { if ( - dependency.priority === 'lazy' || - childAsset.bundleBehavior === 'isolated' // An isolated Dependency, or Bundle must contain all assets it needs to load. + context?.type === 'dependency' && + context?.value.isEntry && + !entries.has(node.value) ) { - let bundleId = bundles.get(childAsset.id); - let bundle; - if (bundleId == null) { - let firstBundleGroup = nullthrows( - bundleGraph.getNode(stack[0][1]), - ); - invariant(firstBundleGroup !== 'root'); - bundle = createBundle({ - asset: childAsset, - target: firstBundleGroup.target, - needsStableName: - dependency.bundleBehavior === 'inline' || - childAsset.bundleBehavior === 'inline' - ? false - : dependency.isEntry || dependency.needsStableName, - bundleBehavior: - dependency.bundleBehavior ?? childAsset.bundleBehavior, - }); - bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); - bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleGroupBundleIds.add(bundleId); - bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - - if ( - // If this dependency requests isolated, but the bundle is not, - // make the bundle isolated for all uses. - dependency.bundleBehavior === 'isolated' && - bundle.bundleBehavior == null - ) { - bundle.bundleBehavior = dependency.bundleBehavior; - } - } + // Skip whole subtrees of other targets by skipping those entries + actions.skipChildren(); + return node; + } + assetToIndex.set(node.value, assets.length); + assets.push(node.value); + + let bundleIdTuple = bundleRoots.get(node.value); + if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { + // Push to the stack (only) when a new bundle is created + stack.push([node.value, bundleIdTuple[0]]); + } else if (bundleIdTuple) { + // Otherwise, push on the last bundle that marks the start of a BundleGroup + stack.push([node.value, stack[stack.length - 1][1]]); + } + } else if (node.type === 'dependency') { + if (context == null) { + return node; + } + let dependency = node.value; + invariant(context?.type === 'asset'); + let parentAsset = context.value; - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependencyPriorityEdges[dependency.priority], - ); - continue; + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return node; } - if ( - parentAsset.type !== childAsset.type || - dependency.priority === 'parallel' || - childAsset.bundleBehavior === 'inline' - ) { - // The referencing bundleRoot is the root of a Bundle that first brings in another bundle (essentially the FIRST parent of a bundle, this may or may not be a bundleGroup) - let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( - stack[stack.length - 1], - ); - let bundleGroup = nullthrows( - bundleGraph.getNode(bundleGroupNodeId), - ); - invariant(bundleGroup !== 'root'); - - let bundleId; - let referencingBundleId = nullthrows( - bundleRoots.get(referencingBundleRoot), - )[0]; - let referencingBundle = nullthrows( - bundleGraph.getNode(referencingBundleId), - ); - invariant(referencingBundle !== 'root'); - let bundle; - bundleId = bundles.get(childAsset.id); - - /** - * If this is an entry bundlegroup, we only allow one bundle per type in those groups - * So attempt to add the asset to the entry bundle if it's of the same type. - * This asset will be created by other dependency if it's in another bundlegroup - * and bundles of other types should be merged in the next step - */ - let bundleGroupRootAsset = nullthrows(bundleGroup.mainEntryAsset); + + for (let childAsset of assets) { if ( - parentAsset.type !== childAsset.type && - entries.has(bundleGroupRootAsset) && - canMerge(bundleGroupRootAsset, childAsset) && - dependency.bundleBehavior == null + dependency.priority === 'lazy' || + childAsset.bundleBehavior === 'isolated' // An isolated Dependency, or Bundle must contain all assets it needs to load. ) { - bundleId = bundleGroupNodeId; - } - if (bundleId == null) { - bundle = createBundle({ - // Bundles created from type changes shouldn't have an entry asset. - asset: childAsset, - type: childAsset.type, - env: childAsset.env, - bundleBehavior: - dependency.bundleBehavior ?? childAsset.bundleBehavior, - target: referencingBundle.target, - needsStableName: - childAsset.bundleBehavior === 'inline' || - dependency.bundleBehavior === 'inline' || - (dependency.priority === 'parallel' && - !dependency.needsStableName) - ? false - : referencingBundle.needsStableName, - }); - bundleId = bundleGraph.addNode(bundle); - - // Store Type-Change bundles for later since we need to know ALL bundlegroups they are part of to reduce/combine them - if (parentAsset.type !== childAsset.type) { - typeChangeIds.add(bundleId); + let bundleId = bundles.get(childAsset.id); + let bundle; + if (bundleId == null) { + let firstBundleGroup = nullthrows( + bundleGraph.getNode(stack[0][1]), + ); + invariant(firstBundleGroup !== 'root'); + bundle = createBundle({ + asset: childAsset, + target: firstBundleGroup.target, + needsStableName: + dependency.bundleBehavior === 'inline' || + childAsset.bundleBehavior === 'inline' + ? false + : dependency.isEntry || dependency.needsStableName, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, + }); + bundleId = bundleGraph.addNode(bundle); + bundles.set(childAsset.id, bundleId); + bundleRoots.set(childAsset, [bundleId, bundleId]); + bundleGroupBundleIds.add(bundleId); + bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } - } else { - bundle = bundleGraph.getNode(bundleId); - invariant(bundle != null && bundle !== 'root'); - if ( - // If this dependency requests isolated, but the bundle is not, - // make the bundle isolated for all uses. - dependency.bundleBehavior === 'isolated' && - bundle.bundleBehavior == null - ) { - bundle.bundleBehavior = dependency.bundleBehavior; - } - } - - bundles.set(childAsset.id, bundleId); - - // A bundle can belong to multiple bundlegroups, all the bundle groups of it's - // ancestors, and all async and entry bundles before it are "bundle groups" - // TODO: We may need to track bundles to all bundleGroups it belongs to in the future. - bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - bundleGraph.addEdge(referencingBundleId, bundleId); - - if (bundleId != bundleGroupNodeId) { dependencyBundleGraph.addEdge( dependencyBundleGraph.addNodeByContentKeyIfNeeded( dependency.id, @@ -578,23 +476,131 @@ function createIdealGraph( type: 'bundle', }, ), - dependencyPriorityEdges.parallel, + dependencyPriorityEdges[dependency.priority], ); + continue; } + if ( + parentAsset.type !== childAsset.type || + dependency.priority === 'parallel' || + childAsset.bundleBehavior === 'inline' + ) { + // The referencing bundleRoot is the root of a Bundle that first brings in another bundle (essentially the FIRST parent of a bundle, this may or may not be a bundleGroup) + let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( + stack[stack.length - 1], + ); + let bundleGroup = nullthrows( + bundleGraph.getNode(bundleGroupNodeId), + ); + invariant(bundleGroup !== 'root'); + + let bundleId; + let referencingBundleId = nullthrows( + bundleRoots.get(referencingBundleRoot), + )[0]; + let referencingBundle = nullthrows( + bundleGraph.getNode(referencingBundleId), + ); + invariant(referencingBundle !== 'root'); + let bundle; + bundleId = bundles.get(childAsset.id); + + /** + * If this is an entry bundlegroup, we only allow one bundle per type in those groups + * So attempt to add the asset to the entry bundle if it's of the same type. + * This asset will be created by other dependency if it's in another bundlegroup + * and bundles of other types should be merged in the next step + */ + let bundleGroupRootAsset = nullthrows(bundleGroup.mainEntryAsset); + if ( + parentAsset.type !== childAsset.type && + entries.has(bundleGroupRootAsset) && + canMerge(bundleGroupRootAsset, childAsset) && + dependency.bundleBehavior == null + ) { + bundleId = bundleGroupNodeId; + } + if (bundleId == null) { + bundle = createBundle({ + // Bundles created from type changes shouldn't have an entry asset. + asset: childAsset, + type: childAsset.type, + env: childAsset.env, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, + target: referencingBundle.target, + needsStableName: + childAsset.bundleBehavior === 'inline' || + dependency.bundleBehavior === 'inline' || + (dependency.priority === 'parallel' && + !dependency.needsStableName) + ? false + : referencingBundle.needsStableName, + }); + bundleId = bundleGraph.addNode(bundle); + + // Store Type-Change bundles for later since we need to know ALL bundlegroups they are part of to reduce/combine them + if (parentAsset.type !== childAsset.type) { + typeChangeIds.add(bundleId); + } + } else { + bundle = bundleGraph.getNode(bundleId); + invariant(bundle != null && bundle !== 'root'); + + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } + } - assetReference.get(childAsset).push([dependency, bundle]); - continue; + bundles.set(childAsset.id, bundleId); + + // A bundle can belong to multiple bundlegroups, all the bundle groups of it's + // ancestors, and all async and entry bundles before it are "bundle groups" + // TODO: We may need to track bundles to all bundleGroups it belongs to in the future. + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + bundleGraph.addEdge(referencingBundleId, bundleId); + + if (bundleId != bundleGroupNodeId) { + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + dependency.id, + { + value: dependency, + type: 'dependency', + }, + ), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependencyPriorityEdges.parallel, + ); + } + + assetReference.get(childAsset).push([dependency, bundle]); + continue; + } } } - } - return node; - }, - exit(node) { - if (stack[stack.length - 1]?.[0] === node.value) { - stack.pop(); - } + return node; + }, + exit(node) { + if (stack[stack.length - 1]?.[0] === node.value) { + stack.pop(); + } + }, }, - }); + null, + {skipUnusedDependencies: true}, + ); let assetSet = BitSet.from(assets); @@ -650,24 +656,31 @@ function createIdealGraph( bundleRootGraph.addNodeByContentKey(root.id, root); // Add in all bundleRoots to BundleRootGraph } } - // ReachableRoots is a Graph of Asset Nodes which represents a BundleRoot, to all assets (non-bundleroot assets + + // ReachableRoots maps bundle roots to all assets (non-bundleroot assets // available to it synchronously (directly) built by traversing the assetgraph once. + let reachableRoots = []; + let reachableAssets = []; + for (let i = 0; i < assets.length; i++) { + reachableRoots[i] = new RawBitSet(assets.length); + reachableAssets[i] = new RawBitSet(assets.length); + } + for (let [root] of bundleRoots) { // Add sync relationships to ReachableRoots - let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); - assetGraph.traverse((node, _, actions) => { - if (node.value === root) { - return; - } - if (node.type === 'dependency') { - let dependency = node.value; - - if (assetGraph.isDependencySkipped(dependency)) { - actions.skipChildren(); + let rootNodeId = nullthrows(assetToIndex.get(root)); + assetGraph.traverse( + (node, _, actions) => { + if (node.value === root) { + return; } + if (node.type === 'dependency') { + let dependency = node.value; - if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if (dependency.priority !== 'sync') { + if ( + dependency.priority !== 'sync' && + dependencyBundleGraph.hasContentKey(dependency.id) + ) { let assets = assetGraph.getDependencyAssets(dependency); if (assets.length === 0) { return; @@ -692,30 +705,25 @@ function createIdealGraph( ); } } - } - if (dependency.priority !== 'sync') { - actions.skipChildren(); + if (dependency.priority !== 'sync') { + actions.skipChildren(); + } + return; } - return; - } - //asset node type - let asset = node.value; - if (asset.bundleBehavior != null || root.type !== asset.type) { - if (root.type !== asset.type && !bundleRoots.has(asset)) { - // A type may not necessarily be a bundleRoot since we've merged at this point - // So we must add that asset in as an island at the very least - reachableRoots.addNodeByContentKeyIfNeeded(node.value.id, node.value); + //asset node type + let asset = node.value; + if (asset.bundleBehavior != null || root.type !== asset.type) { + actions.skipChildren(); + return; } - actions.skipChildren(); - return; - } - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - reachableRoots.addEdge(rootNodeId, nodeId); - }, root); + let nodeId = nullthrows(assetToIndex.get(node.value)); + reachableAssets[rootNodeId].add(nodeId); + reachableRoots[nodeId].add(rootNodeId); + }, + root, + {skipUnusedDependencies: true}, + ); } // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets @@ -765,15 +773,10 @@ function createIdealGraph( for (let bundleRoot of bundleInGroup.assets) { // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - - for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { - available.add(asset); - } + available.add(bundleRoot); + available.unionRaw( + reachableAssets[nullthrows(assetToIndex.get(bundleRoot))], + ); } } } @@ -816,13 +819,9 @@ function createIdealGraph( ancestorAssets.set(child, currentChildAvailable.clone()); } if (isParallel) { - for (let reachableNodeId of reachableRoots.getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(child.id), - )) { - let asset = nullthrows(reachableRoots.getNode(reachableNodeId)); - - parallelAvailability.add(asset); - } + parallelAvailability.unionRaw( + reachableAssets[nullthrows(assetToIndex.get(child))], + ); parallelAvailability.add(child); //The next sibling should have older sibling available via parallel } } @@ -845,9 +844,8 @@ function createIdealGraph( continue; } if ( - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey(parent.id), - reachableRoots.getNodeIdByContentKey(bundleRoot.id), + reachableAssets[nullthrows(assetToIndex.get(parent))].has( + nullthrows(assetToIndex.get(bundleRoot)), ) || ancestorAssets.get(parent)?.has(bundleRoot) ) { @@ -868,36 +866,37 @@ function createIdealGraph( deleteBundle(bundleRoot); } } + // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. // This creates a maximally code split bundle graph with no duplication. - for (let asset of assets) { - // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means - let reachable: Array = getReachableBundleRoots( - asset, - reachableRoots, - ).reverse(); - - let reachableEntries = []; - let reachableNonEntries = []; + let reachable = new RawBitSet(assets.length); + let reachableNonEntries = new RawBitSet(assets.length); + let reachableIntersection = new RawBitSet(assets.length); + for (let i = 0; i < assets.length; i++) { + let asset = assets[i]; if (asset.meta.isConstantModule === true) { // Add assets to non-splittable bundles. - for (let entry of reachable) { + reachableRoots[i].forEach(nodeId => { + let entry = assets[nodeId]; let entryBundleId = nullthrows(bundleRoots.get(entry))[0]; let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); invariant(entryBundle !== 'root'); entryBundle.assets.add(asset); entryBundle.size += asset.stats.size; - } + }); continue; } + // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means. // Filter out entries, since they can't have shared bundles. // Neither can non-splittable, isolated, or needing of stable name bundles. // Reserve those filtered out bundles since we add the asset back into them. - for (let a of reachable) { + reachableNonEntries.clear(); + reachableRoots[i].forEach(nodeId => { + let a = assets[nodeId]; if ( entries.has(a) || !a.isBundleSplittable || @@ -905,16 +904,18 @@ function createIdealGraph( (getBundleFromBundleRoot(a).needsStableName || getBundleFromBundleRoot(a).bundleBehavior === 'isolated')) ) { - reachableEntries.push(a); - } else { - reachableNonEntries.push(a); + // Add asset to non-splittable bundles. + let entryBundleId = nullthrows(bundleRoots.get(a))[0]; + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + invariant(entryBundle !== 'root'); + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; + } else if (!ancestorAssets.get(a)?.has(asset)) { + // Filter out bundles from this asset's reachable array if + // bundle does not contain the asset in its ancestry + reachableNonEntries.add(nodeId); } - } - reachable = reachableNonEntries; - - // Filter out bundles from this asset's reachable array if - // bundle does not contain the asset in its ancestry - reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); + }); // Finally, filter out bundleRoots (bundles) from this assets // reachable if they are subgraphs, and reuse that subgraph bundle @@ -924,18 +925,19 @@ function createIdealGraph( // a bundle represents the exact set of assets a set of bundles would share // if a bundle b is a subgraph of another bundle f, reuse it, drawing an edge between the two - let canReuse: Set = new Set(); + reachable.bits.set(reachableNonEntries.bits); if (config.disableSharedBundles === false) { - for (let candidateSourceBundleRoot of reachable) { + reachableNonEntries.forEach(candidateId => { + let candidateSourceBundleRoot = assets[candidateId]; let candidateSourceBundleId = nullthrows( bundleRoots.get(candidateSourceBundleRoot), )[0]; if (candidateSourceBundleRoot.env.isIsolated()) { - continue; + return; } let reuseableBundleId = bundles.get(asset.id); if (reuseableBundleId != null) { - canReuse.add(candidateSourceBundleRoot); + reachable.delete(candidateId); bundleGraph.addEdge(candidateSourceBundleId, reuseableBundleId); let reusableBundle = bundleGraph.getNode(reuseableBundleId); @@ -944,49 +946,41 @@ function createIdealGraph( } else { // Asset is not a bundleRoot, but if its ancestor bundle (in the asset's reachable) can be // reused as a subgraph of another bundleRoot in its reachable, reuse it - for (let otherReuseCandidate of reachable) { - if (candidateSourceBundleRoot === otherReuseCandidate) continue; - let reusableCandidateReachable = getReachableBundleRoots( - otherReuseCandidate, - reachableRoots, - ).filter(b => !ancestorAssets.get(b)?.has(otherReuseCandidate)); - if ( - reusableCandidateReachable.includes(candidateSourceBundleRoot) - ) { - let reusableBundleId = nullthrows( - bundles.get(otherReuseCandidate.id), - ); - canReuse.add(candidateSourceBundleRoot); - bundleGraph.addEdge( - nullthrows(bundles.get(candidateSourceBundleRoot.id)), - reusableBundleId, - ); - let reusableBundle = bundleGraph.getNode(reusableBundleId); - invariant(reusableBundle !== 'root' && reusableBundle != null); - reusableBundle.sourceBundles.add(candidateSourceBundleId); - } - } + reachableIntersection.bits.set(reachableNonEntries.bits); + reachableIntersection.intersect(reachableAssets[candidateId]); + reachableIntersection.forEach(otherCandidateId => { + let otherReuseCandidate = assets[otherCandidateId]; + if (candidateSourceBundleRoot === otherReuseCandidate) return; + let reusableBundleId = nullthrows( + bundles.get(otherReuseCandidate.id), + ); + reachable.delete(candidateId); + bundleGraph.addEdge( + nullthrows(bundles.get(candidateSourceBundleRoot.id)), + reusableBundleId, + ); + let reusableBundle = bundleGraph.getNode(reusableBundleId); + invariant(reusableBundle !== 'root' && reusableBundle != null); + reusableBundle.sourceBundles.add(candidateSourceBundleId); + }); } - } - } - //Bundles that are reused should not be considered for shared bundles, so filter them out - reachable = reachable.filter(b => !canReuse.has(b)); - - // Add assets to non-splittable bundles. - for (let entry of reachableEntries) { - let entryBundleId = nullthrows(bundleRoots.get(entry))[0]; - let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); - invariant(entryBundle !== 'root'); - entryBundle.assets.add(asset); - entryBundle.size += asset.stats.size; + }); } + let reachableArray = []; + reachable.forEach(id => { + reachableArray.push(assets[id]); + }); + + // Create shared bundles for splittable bundles. if ( config.disableSharedBundles === false && - reachable.length > config.minBundles + reachableArray.length > config.minBundles ) { - let sourceBundles = reachable.map(a => nullthrows(bundleRoots.get(a))[0]); - let key = reachable.map(a => a.id).join(','); + let sourceBundles = reachableArray.map( + a => nullthrows(bundleRoots.get(a))[0], + ); + let key = reachableArray.map(a => a.id).join(','); let bundleId = bundles.get(key); let bundle; if (bundleId == null) { @@ -1037,9 +1031,9 @@ function createIdealGraph( }); } else if ( config.disableSharedBundles === true || - reachable.length <= config.minBundles + reachableArray.length <= config.minBundles ) { - for (let root of reachable) { + for (let root of reachableArray) { let bundle = nullthrows( bundleGraph.getNode(nullthrows(bundleRoots.get(root))[0]), ); @@ -1448,12 +1442,6 @@ async function loadBundlerConfig( }; } -function getReachableBundleRoots(asset, graph): Array { - return graph - .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) - .map(nodeId => nullthrows(graph.getNode(nodeId))); -} - function getEntryByTarget( bundleGraph: MutableBundleGraph, ): DefaultMap> { diff --git a/packages/core/core/src/public/BundleGraph.js b/packages/core/core/src/public/BundleGraph.js index 1044a0ad9cd..39d12920d42 100644 --- a/packages/core/core/src/public/BundleGraph.js +++ b/packages/core/core/src/public/BundleGraph.js @@ -259,18 +259,27 @@ export default class BundleGraph traverse( visit: GraphVisitor, start?: ?IAsset, + opts?: ?{|skipUnusedDependencies?: boolean|}, ): ?TContext { return this.#graph.traverse( - mapVisitor( - node => - node.type === 'asset' - ? {type: 'asset', value: assetFromValue(node.value, this.#options)} - : { - type: 'dependency', - value: new Dependency(node.value, this.#options), - }, - visit, - ), + mapVisitor((node, actions) => { + // Skipping unused dependencies here is faster than doing an isDependencySkipped check inside the visitor + // because the node needs to be re-looked up by id from the hashmap. + if ( + opts?.skipUnusedDependencies && + node.type === 'dependency' && + (node.hasDeferred || node.excluded) + ) { + actions.skipChildren(); + return null; + } + return node.type === 'asset' + ? {type: 'asset', value: assetFromValue(node.value, this.#options)} + : { + type: 'dependency', + value: getPublicDependency(node.value, this.#options), + }; + }, visit), start ? assetToAssetValue(start) : undefined, ); } From 3accc8428c0cfda80af24b2cedd4c8a3a4da1a6e Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 23 Sep 2023 23:18:00 -0700 Subject: [PATCH 6/8] fix tests --- packages/core/core/test/PublicDependency.test.js | 6 +++--- packages/core/graph/test/Graph.test.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core/core/test/PublicDependency.test.js b/packages/core/core/test/PublicDependency.test.js index 914e48968e9..23401847015 100644 --- a/packages/core/core/test/PublicDependency.test.js +++ b/packages/core/core/test/PublicDependency.test.js @@ -3,7 +3,7 @@ import assert from 'assert'; import {createEnvironment} from '../src/Environment'; import {createDependency} from '../src/Dependency'; -import Dependency from '../src/public/Dependency'; +import {getPublicDependency} from '../src/public/Dependency'; import {DEFAULT_OPTIONS} from './test-utils'; describe('Public Dependency', () => { @@ -15,8 +15,8 @@ describe('Public Dependency', () => { }); assert.equal( - new Dependency(internalDependency, DEFAULT_OPTIONS), - new Dependency(internalDependency, DEFAULT_OPTIONS), + getPublicDependency(internalDependency, DEFAULT_OPTIONS), + getPublicDependency(internalDependency, DEFAULT_OPTIONS), ); }); }); diff --git a/packages/core/graph/test/Graph.test.js b/packages/core/graph/test/Graph.test.js index 7a045dd9b97..faf9cbc27e8 100644 --- a/packages/core/graph/test/Graph.test.js +++ b/packages/core/graph/test/Graph.test.js @@ -9,7 +9,7 @@ import {toNodeId} from '../src/types'; describe('Graph', () => { it('constructor should initialize an empty graph', () => { let graph = new Graph(); - assert.deepEqual(graph.nodes, new Map()); + assert.deepEqual(graph.nodes, []); assert.deepEqual([...graph.getAllEdges()], []); }); @@ -164,7 +164,7 @@ describe('Graph', () => { graph.removeNode(nodeB); - assert.deepEqual([...graph.nodes.keys()], [nodeA, nodeC, nodeF]); + assert.deepEqual(graph.nodes.filter(Boolean), ['a', 'c', 'f']); assert.deepEqual(Array.from(graph.getAllEdges()), [ {from: nodeA, to: nodeC, type: 1}, {from: nodeC, to: nodeF, type: 1}, @@ -209,7 +209,7 @@ describe('Graph', () => { graph.removeNode(nodeB); - assert.deepEqual([...graph.nodes.keys()], [nodeA, nodeC, nodeF]); + assert.deepEqual(graph.nodes.filter(Boolean), ['a', 'c', 'f']); assert.deepEqual(Array.from(graph.getAllEdges()), [ {from: nodeA, to: nodeC, type: 1}, {from: nodeC, to: nodeF, type: 1}, @@ -337,7 +337,7 @@ describe('Graph', () => { graph.removeNode(node1); - assert.strictEqual(graph.nodes.length, 1); + assert.deepEqual(graph.nodes.filter(Boolean), ['root']); assert.deepStrictEqual(Array.from(graph.getAllEdges()), []); }); }); From 62f2f766c91c785ec8d5470cc8a32071cf8ba23d Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sun, 24 Sep 2023 02:11:13 -0700 Subject: [PATCH 7/8] Optimize memory usage --- .../bundlers/default/src/DefaultBundler.js | 111 ++++++++++-------- packages/core/graph/src/Graph.js | 4 +- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 4b90178c9a1..04d122c67a2 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -331,17 +331,16 @@ function createIdealGraph( parallel: 1, lazy: 2, }; - // ContentGraph that models bundleRoots, with parallel & async deps only to inform reachability - let bundleRootGraph: ContentGraph< - BundleRoot | 'root', + // Graph that models bundleRoots, with parallel & async deps only to inform reachability + let bundleRootGraph: Graph< + number, // asset index $Values, - > = new ContentGraph(); + > = new Graph(); + let assetToBundleRootNodeId = new Map(); let bundleGroupBundleIds: Set = new Set(); - let rootNodeId = nullthrows(bundleRootGraph.addNode('root')); let bundleGraphRootNodeId = nullthrows(bundleGraph.addNode('root')); - bundleRootGraph.setRootNodeId(rootNodeId); bundleGraph.setRootNodeId(bundleGraphRootNodeId); // Step Create Entry Bundles for (let [asset, dependency] of entries) { @@ -353,10 +352,6 @@ function createIdealGraph( let nodeId = bundleGraph.addNode(bundle); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); - bundleRootGraph.addEdge( - rootNodeId, - bundleRootGraph.addNodeByContentKey(asset.id, asset), - ); bundleGraph.addEdge(bundleGraphRootNodeId, nodeId); dependencyBundleGraph.addEdge( @@ -651,24 +646,37 @@ function createIdealGraph( * The two graphs, are used to build up ancestorAssets, a structure which holds all availability by * all means for each asset. */ + + let rootNodeId = bundleRootGraph.addNode(-1); + bundleRootGraph.setRootNodeId(rootNodeId); + for (let [root] of bundleRoots) { - if (!entries.has(root)) { - bundleRootGraph.addNodeByContentKey(root.id, root); // Add in all bundleRoots to BundleRootGraph + let nodeId = bundleRootGraph.addNode(nullthrows(assetToIndex.get(root))); + assetToBundleRootNodeId.set(root, nodeId); + if (entries.has(root)) { + bundleRootGraph.addEdge(rootNodeId, nodeId); } } - // ReachableRoots maps bundle roots to all assets (non-bundleroot assets - // available to it synchronously (directly) built by traversing the assetgraph once. + // reachableRoots is an array of bit sets for each asset. Each bit set + // indicates which bundle roots are reachable from that asset synchronously. let reachableRoots = []; - let reachableAssets = []; for (let i = 0; i < assets.length; i++) { - reachableRoots[i] = new RawBitSet(assets.length); - reachableAssets[i] = new RawBitSet(assets.length); + reachableRoots.push(new RawBitSet(bundleRootGraph.nodes.length)); } - for (let [root] of bundleRoots) { + // reachableAssets is the inverse mapping of reachableRoots. For each bundle root, + // it contains a bit set that indicates which assets are reachable from it. + let reachableAssets = []; + + for (let [bundleRootId, assetId] of bundleRootGraph.nodes.entries()) { + let reachable = new RawBitSet(assets.length); + reachableAssets.push(reachable); + + if (bundleRootId == rootNodeId || assetId == null) continue; + // Add sync relationships to ReachableRoots - let rootNodeId = nullthrows(assetToIndex.get(root)); + let root = assets[assetId]; assetGraph.traverse( (node, _, actions) => { if (node.value === root) { @@ -697,8 +705,8 @@ function createIdealGraph( bundle.env.context === root.env.context ) { bundleRootGraph.addEdge( - bundleRootGraph.getNodeIdByContentKey(root.id), - bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + bundleRootId, + nullthrows(assetToBundleRootNodeId.get(bundleRoot)), dependency.priority === 'parallel' ? bundleRootEdgeTypes.parallel : bundleRootEdgeTypes.lazy, @@ -717,9 +725,9 @@ function createIdealGraph( actions.skipChildren(); return; } - let nodeId = nullthrows(assetToIndex.get(node.value)); - reachableAssets[rootNodeId].add(nodeId); - reachableRoots[nodeId].add(rootNodeId); + let assetIndex = nullthrows(assetToIndex.get(node.value)); + reachable.add(assetIndex); + reachableRoots[assetIndex].add(bundleRootId); }, root, {skipUnusedDependencies: true}, @@ -744,9 +752,8 @@ function createIdealGraph( // to all assets available to it (meaning they will exist guaranteed when the bundleRoot is loaded) // The topological sort ensures all parents are visited before the node we want to process. for (let nodeId of bundleRootGraph.topoSort(ALL_EDGE_TYPES)) { - const bundleRoot = bundleRootGraph.getNode(nodeId); - if (bundleRoot === 'root') continue; - invariant(bundleRoot != null); + if (nodeId === rootNodeId) continue; + const bundleRoot = assets[nullthrows(bundleRootGraph.getNode(nodeId))]; let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; // At a BundleRoot, we access it's available assets (via ancestorAssets), @@ -775,7 +782,9 @@ function createIdealGraph( // Assets directly connected to current bundleRoot available.add(bundleRoot); available.unionRaw( - reachableAssets[nullthrows(assetToIndex.get(bundleRoot))], + reachableAssets[ + nullthrows(assetToBundleRootNodeId.get(bundleRoot)) + ], ); } } @@ -792,8 +801,7 @@ function createIdealGraph( let parallelAvailability = assetSet.cloneEmpty(); for (let childId of children) { - let child = bundleRootGraph.getNode(childId); - invariant(child !== 'root' && child != null); + let child = assets[nullthrows(bundleRootGraph.getNode(childId))]; let bundleBehavior = getBundleFromBundleRoot(child).bundleBehavior; if (bundleBehavior != null) { continue; @@ -820,7 +828,7 @@ function createIdealGraph( } if (isParallel) { parallelAvailability.unionRaw( - reachableAssets[nullthrows(assetToIndex.get(child))], + reachableAssets[nullthrows(assetToBundleRootNodeId.get(child))], ); parallelAvailability.add(child); //The next sibling should have older sibling available via parallel } @@ -830,21 +838,23 @@ function createIdealGraph( // the bundle is synchronously available elsewhere. // We can query sync assets available via reachableRoots. If the parent has // the bundleRoot by reachableRoots AND ancestorAssets, internalize it. - for (let [id, bundleRoot] of bundleRootGraph.nodes.entries()) { - if (!bundleRoot || bundleRoot === 'root') continue; - let parentRoots = bundleRootGraph - .getNodeIdsConnectedTo(id, ALL_EDGE_TYPES) - .map(id => nullthrows(bundleRootGraph.getNode(id))); + for (let [id, bundleRootId] of bundleRootGraph.nodes.entries()) { + if (bundleRootId == null || id === rootNodeId) continue; + let bundleRoot = assets[bundleRootId]; + let parentRoots = bundleRootGraph.getNodeIdsConnectedTo(id, ALL_EDGE_TYPES); let canDelete = getBundleFromBundleRoot(bundleRoot).bundleBehavior !== 'isolated'; if (parentRoots.length === 0) continue; - for (let parent of parentRoots) { - if (parent === 'root') { + for (let parentId of parentRoots) { + if (parentId === rootNodeId) { + // connected to root. canDelete = false; continue; } + let parentAssetId = nullthrows(bundleRootGraph.getNode(parentId)); + let parent = assets[parentAssetId]; if ( - reachableAssets[nullthrows(assetToIndex.get(parent))].has( + reachableAssets[parentId].has( nullthrows(assetToIndex.get(bundleRoot)), ) || ancestorAssets.get(parent)?.has(bundleRoot) @@ -879,7 +889,9 @@ function createIdealGraph( if (asset.meta.isConstantModule === true) { // Add assets to non-splittable bundles. reachableRoots[i].forEach(nodeId => { - let entry = assets[nodeId]; + let assetId = bundleRootGraph.getNode(nodeId); + if (assetId == null) return; // deleted + let entry = assets[assetId]; let entryBundleId = nullthrows(bundleRoots.get(entry))[0]; let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); invariant(entryBundle !== 'root'); @@ -896,7 +908,9 @@ function createIdealGraph( // Reserve those filtered out bundles since we add the asset back into them. reachableNonEntries.clear(); reachableRoots[i].forEach(nodeId => { - let a = assets[nodeId]; + let assetId = bundleRootGraph.getNode(nodeId); + if (assetId == null) return; // deleted + let a = assets[assetId]; if ( entries.has(a) || !a.isBundleSplittable || @@ -913,7 +927,7 @@ function createIdealGraph( } else if (!ancestorAssets.get(a)?.has(asset)) { // Filter out bundles from this asset's reachable array if // bundle does not contain the asset in its ancestry - reachableNonEntries.add(nodeId); + reachableNonEntries.add(assetId); } }); @@ -947,7 +961,11 @@ function createIdealGraph( // Asset is not a bundleRoot, but if its ancestor bundle (in the asset's reachable) can be // reused as a subgraph of another bundleRoot in its reachable, reuse it reachableIntersection.bits.set(reachableNonEntries.bits); - reachableIntersection.intersect(reachableAssets[candidateId]); + reachableIntersection.intersect( + reachableAssets[ + nullthrows(assetToBundleRootNodeId.get(candidateSourceBundleRoot)) + ], + ); reachableIntersection.forEach(otherCandidateId => { let otherReuseCandidate = assets[otherCandidateId]; if (candidateSourceBundleRoot === otherReuseCandidate) return; @@ -1178,10 +1196,9 @@ function createIdealGraph( bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); bundleRoots.delete(bundleRoot); bundles.delete(bundleRoot.id); - if (bundleRootGraph.hasContentKey(bundleRoot.id)) { - bundleRootGraph.removeNode( - bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - ); + let bundleRootId = assetToBundleRootNodeId.get(bundleRoot); + if (bundleRootId != null && bundleRootGraph.hasNode(bundleRootId)) { + bundleRootGraph.removeNode(bundleRootId); } } function getBundleGroupsForBundle(nodeId: NodeId) { diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index 1f3c991a833..ed3520e3071 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -95,11 +95,11 @@ export default class Graph { throw new Error(`Edge type "${type}" not allowed`); } - if (!this.getNode(from)) { + if (this.getNode(from) == null) { throw new Error(`"from" node '${fromNodeId(from)}' not found`); } - if (!this.getNode(to)) { + if (this.getNode(to) == null) { throw new Error(`"to" node '${fromNodeId(to)}' not found`); } From 2f123245d399f309848aee4810069073695b9503 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 30 Sep 2023 14:57:07 -0700 Subject: [PATCH 8/8] Move BitSet into graph package and refactor bundler to only have one version --- .../bundlers/default/src/DefaultBundler.js | 88 ++++---- packages/core/graph/src/BitSet.js | 93 +++++++++ packages/core/graph/src/Graph.js | 8 +- packages/core/graph/src/index.js | 1 + .../core/{utils => graph}/test/BitSet.test.js | 49 ++--- packages/core/utils/src/BitSet.js | 190 ------------------ packages/core/utils/src/index.js | 1 - 7 files changed, 155 insertions(+), 275 deletions(-) create mode 100644 packages/core/graph/src/BitSet.js rename packages/core/{utils => graph}/test/BitSet.test.js (61%) delete mode 100644 packages/core/utils/src/BitSet.js diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 04d122c67a2..792530525da 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -14,19 +14,13 @@ import type { } from '@parcel/types'; import type {NodeId} from '@parcel/graph'; import type {SchemaEntity} from '@parcel/utils'; -import {ContentGraph, Graph} from '@parcel/graph'; +import {ContentGraph, Graph, BitSet} from '@parcel/graph'; import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; import logger from '@parcel/logger'; -import { - setEqual, - validateSchema, - DefaultMap, - BitSet, - RawBitSet, -} from '@parcel/utils'; +import {setEqual, validateSchema, DefaultMap} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -66,7 +60,7 @@ type BundleRoot = Asset; export type Bundle = {| uniqueKey: ?string, assets: Set, - internalizedAssets?: BitSet, + internalizedAssets?: BitSet, bundleBehavior?: ?BundleBehavior, needsStableName: boolean, mainEntryAsset: ?Asset, @@ -98,6 +92,7 @@ type DependencyBundleGraph = ContentGraph< // which mutates the assetGraph into the bundleGraph we would // expect from default bundler type IdealGraph = {| + assets: Array, dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, bundleGroupBundleIds: Set, @@ -237,8 +232,10 @@ function decorateLegacyGraph( if (!idealBundle || idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); if (idealBundle.internalizedAssets) { - for (let internalized of idealBundle.internalizedAssets.values()) { - let incomingDeps = bundleGraph.getIncomingDependencies(internalized); + idealBundle.internalizedAssets.forEach(internalized => { + let incomingDeps = bundleGraph.getIncomingDependencies( + idealGraph.assets[internalized], + ); for (let incomingDep of incomingDeps) { if ( incomingDep.priority === 'lazy' && @@ -248,7 +245,7 @@ function decorateLegacyGraph( bundleGraph.internalizeAsyncDependency(bundle, incomingDep); } } - } + }); } } @@ -597,8 +594,6 @@ function createIdealGraph( {skipUnusedDependencies: true}, ); - let assetSet = BitSet.from(assets); - // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups for (let [nodeIdA, a] of bundleGraph.nodes.entries()) { //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes @@ -662,16 +657,23 @@ function createIdealGraph( // indicates which bundle roots are reachable from that asset synchronously. let reachableRoots = []; for (let i = 0; i < assets.length; i++) { - reachableRoots.push(new RawBitSet(bundleRootGraph.nodes.length)); + reachableRoots.push(new BitSet(bundleRootGraph.nodes.length)); } // reachableAssets is the inverse mapping of reachableRoots. For each bundle root, // it contains a bit set that indicates which assets are reachable from it. let reachableAssets = []; + // ancestorAssets maps bundle roots to the set of all assets available to it at runtime, + // including in earlier parallel bundles. These are intersected through all paths to + // the bundle to ensure that the available assets are always present no matter in which + // order the bundles are loaded. + let ancestorAssets = []; + for (let [bundleRootId, assetId] of bundleRootGraph.nodes.entries()) { - let reachable = new RawBitSet(assets.length); + let reachable = new BitSet(assets.length); reachableAssets.push(reachable); + ancestorAssets.push(null); if (bundleRootId == rootNodeId || assetId == null) continue; @@ -733,13 +735,11 @@ function createIdealGraph( {skipUnusedDependencies: true}, ); } - // Maps a given bundleRoot to the assets reachable from it, - // and the bundleRoots reachable from each of these assets - let ancestorAssets: Map> = new Map(); for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries - ancestorAssets.set(entry, assetSet.cloneEmpty()); + let entryId = nullthrows(assetToBundleRootNodeId.get(entry)); + ancestorAssets[entryId] = new BitSet(assets.length); } // Step Determine Availability @@ -765,9 +765,9 @@ function createIdealGraph( // it belongs to. It's the intersection of those sets. let available; if (bundleRoot.bundleBehavior === 'isolated') { - available = assetSet.cloneEmpty(); + available = new BitSet(assets.length); } else { - available = nullthrows(ancestorAssets.get(bundleRoot)).clone(); + available = nullthrows(ancestorAssets[nodeId]).clone(); for (let bundleIdInGroup of [ bundleGroupId, ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), @@ -780,8 +780,8 @@ function createIdealGraph( for (let bundleRoot of bundleInGroup.assets) { // Assets directly connected to current bundleRoot - available.add(bundleRoot); - available.unionRaw( + available.add(nullthrows(assetToIndex.get(bundleRoot))); + available.union( reachableAssets[ nullthrows(assetToBundleRootNodeId.get(bundleRoot)) ], @@ -798,10 +798,11 @@ function createIdealGraph( nodeId, ALL_EDGE_TYPES, ); - let parallelAvailability = assetSet.cloneEmpty(); + let parallelAvailability = new BitSet(assets.length); for (let childId of children) { - let child = assets[nullthrows(bundleRootGraph.getNode(childId))]; + let assetId = nullthrows(bundleRootGraph.getNode(childId)); + let child = assets[assetId]; let bundleBehavior = getBundleFromBundleRoot(child).bundleBehavior; if (bundleBehavior != null) { continue; @@ -817,20 +818,18 @@ function createIdealGraph( // intersect the availability built there with the previously computed // availability. this ensures no matter which bundleGroup loads a particular bundle, // it will only assume availability of assets it has under any circumstance - const childAvailableAssets = ancestorAssets.get(child); + const childAvailableAssets = ancestorAssets[childId]; let currentChildAvailable = isParallel ? BitSet.union(parallelAvailability, available) : available; if (childAvailableAssets != null) { childAvailableAssets.intersect(currentChildAvailable); } else { - ancestorAssets.set(child, currentChildAvailable.clone()); + ancestorAssets[childId] = currentChildAvailable.clone(); } if (isParallel) { - parallelAvailability.unionRaw( - reachableAssets[nullthrows(assetToBundleRootNodeId.get(child))], - ); - parallelAvailability.add(child); //The next sibling should have older sibling available via parallel + parallelAvailability.union(reachableAssets[childId]); + parallelAvailability.add(assetId); //The next sibling should have older sibling available via parallel } } } @@ -851,23 +850,21 @@ function createIdealGraph( canDelete = false; continue; } - let parentAssetId = nullthrows(bundleRootGraph.getNode(parentId)); - let parent = assets[parentAssetId]; if ( - reachableAssets[parentId].has( - nullthrows(assetToIndex.get(bundleRoot)), - ) || - ancestorAssets.get(parent)?.has(bundleRoot) + reachableAssets[parentId].has(bundleRootId) || + ancestorAssets[parentId]?.has(bundleRootId) ) { + let parentAssetId = nullthrows(bundleRootGraph.getNode(parentId)); + let parent = assets[parentAssetId]; let parentBundle = bundleGraph.getNode( nullthrows(bundles.get(parent.id)), ); invariant(parentBundle != null && parentBundle !== 'root'); if (!parentBundle.internalizedAssets) { - parentBundle.internalizedAssets = assetSet.cloneEmpty(); + parentBundle.internalizedAssets = new BitSet(assets.length); } - parentBundle.internalizedAssets.add(bundleRoot); + parentBundle.internalizedAssets.add(bundleRootId); } else { canDelete = false; } @@ -880,9 +877,9 @@ function createIdealGraph( // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. // This creates a maximally code split bundle graph with no duplication. - let reachable = new RawBitSet(assets.length); - let reachableNonEntries = new RawBitSet(assets.length); - let reachableIntersection = new RawBitSet(assets.length); + let reachable = new BitSet(assets.length); + let reachableNonEntries = new BitSet(assets.length); + let reachableIntersection = new BitSet(assets.length); for (let i = 0; i < assets.length; i++) { let asset = assets[i]; @@ -924,7 +921,7 @@ function createIdealGraph( invariant(entryBundle !== 'root'); entryBundle.assets.add(asset); entryBundle.size += asset.stats.size; - } else if (!ancestorAssets.get(a)?.has(asset)) { + } else if (!ancestorAssets[nodeId]?.has(i)) { // Filter out bundles from this asset's reachable array if // bundle does not contain the asset in its ancestry reachableNonEntries.add(assetId); @@ -1014,7 +1011,7 @@ function createIdealGraph( bundle.sourceBundles = new Set(sourceBundles); let sharedInternalizedAssets = firstSourceBundle.internalizedAssets ? firstSourceBundle.internalizedAssets.clone() - : assetSet.cloneEmpty(); + : new BitSet(assets.length); for (let p of sourceBundles) { let parentBundle = nullthrows(bundleGraph.getNode(p)); @@ -1295,6 +1292,7 @@ function createIdealGraph( } return { + assets, bundleGraph, dependencyBundleGraph, bundleGroupBundleIds, diff --git a/packages/core/graph/src/BitSet.js b/packages/core/graph/src/BitSet.js new file mode 100644 index 00000000000..21cb6e66a33 --- /dev/null +++ b/packages/core/graph/src/BitSet.js @@ -0,0 +1,93 @@ +// @flow strict-local + +// Small wasm program that exposes the `ctz` instruction. +// https://developer.mozilla.org/en-US/docs/WebAssembly/Reference/Numeric/Count_trailing_zeros +const wasmBuf = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, 0x01, + 0x7f, 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0d, 0x01, 0x09, 0x74, 0x72, + 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x30, 0x00, 0x00, 0x0a, 0x07, 0x01, 0x05, + 0x00, 0x20, 0x00, 0x68, 0x0b, 0x00, 0x0f, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x02, + 0x08, 0x01, 0x00, 0x01, 0x00, 0x03, 0x6e, 0x75, 0x6d, +]); + +// eslint-disable-next-line +const {trailing0} = new WebAssembly.Instance(new WebAssembly.Module(wasmBuf)) + .exports; + +export class BitSet { + bits: Uint32Array; + + constructor(maxBits: number) { + this.bits = new Uint32Array(Math.ceil(maxBits / 32)); + } + + clone(): BitSet { + let res = new BitSet(this.capacity); + res.bits.set(this.bits); + return res; + } + + static union(a: BitSet, b: BitSet): BitSet { + let res = a.clone(); + res.union(b); + return res; + } + + get capacity(): number { + return this.bits.length * 32; + } + + add(bit: number) { + let i = bit >>> 5; + let b = bit & 31; + this.bits[i] |= 1 << b; + } + + delete(bit: number) { + let i = bit >>> 5; + let b = bit & 31; + this.bits[i] &= ~(1 << b); + } + + has(bit: number): boolean { + let i = bit >>> 5; + let b = bit & 31; + return Boolean(this.bits[i] & (1 << b)); + } + + clear() { + this.bits.fill(0); + } + + intersect(other: BitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] &= other.bits[i]; + } + } + + union(other: BitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] |= other.bits[i]; + } + } + + remove(other: BitSet) { + for (let i = 0; i < this.bits.length; i++) { + this.bits[i] &= ~other.bits[i]; + } + } + + forEach(fn: (bit: number) => void) { + // https://lemire.me/blog/2018/02/21/iterating-over-set-bits-quickly/ + let bits = this.bits; + for (let k = 0; k < bits.length; k++) { + let v = bits[k]; + while (v !== 0) { + let t = (v & -v) >>> 0; + // $FlowFixMe + fn((k << 5) + trailing0(v)); + v ^= t; + } + } + } +} diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index ed3520e3071..7abcccfb1a6 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -8,7 +8,7 @@ import type { GraphVisitor, GraphTraversalCallback, } from '@parcel/types'; -import {RawBitSet} from '@parcel/utils'; +import {BitSet} from './BitSet'; import nullthrows from 'nullthrows'; @@ -32,7 +32,7 @@ export default class Graph { nodes: Array; adjacencyList: AdjacencyList; rootNodeId: ?NodeId; - _visited: ?RawBitSet; + _visited: ?BitSet; constructor(opts: ?GraphOpts) { this.nodes = opts?.nodes || []; @@ -334,7 +334,7 @@ export default class Graph { let visited; if (!this._visited || this._visited.capacity < this.nodes.length) { - this._visited = new RawBitSet(this.nodes.length); + this._visited = new BitSet(this.nodes.length); visited = this._visited; } else { visited = this._visited; @@ -405,7 +405,7 @@ export default class Graph { let visited; if (!this._visited || this._visited.capacity < this.nodes.length) { - this._visited = new RawBitSet(this.nodes.length); + this._visited = new BitSet(this.nodes.length); visited = this._visited; } else { visited = this._visited; diff --git a/packages/core/graph/src/index.js b/packages/core/graph/src/index.js index 0eaafecc4ed..1927aeca7d6 100644 --- a/packages/core/graph/src/index.js +++ b/packages/core/graph/src/index.js @@ -6,3 +6,4 @@ export type {ContentGraphOpts, SerializedContentGraph} from './ContentGraph'; export {toNodeId, fromNodeId} from './types'; export {default as Graph, ALL_EDGE_TYPES, mapVisitor} from './Graph'; export {default as ContentGraph} from './ContentGraph'; +export {BitSet} from './BitSet'; diff --git a/packages/core/utils/test/BitSet.test.js b/packages/core/graph/test/BitSet.test.js similarity index 61% rename from packages/core/utils/test/BitSet.test.js rename to packages/core/graph/test/BitSet.test.js index c21656c37a1..26134710a93 100644 --- a/packages/core/utils/test/BitSet.test.js +++ b/packages/core/graph/test/BitSet.test.js @@ -3,8 +3,11 @@ import assert from 'assert'; import {BitSet} from '../src/BitSet'; -function assertValues(set: BitSet, values: Array) { - let setValues = set.values(); +function assertValues(set: BitSet, values: Array) { + let setValues = []; + set.forEach(bit => { + setValues.push(bit); + }); for (let value of values) { assert(set.has(value), 'Set.has returned false'); @@ -21,18 +24,8 @@ function assertValues(set: BitSet, values: Array) { } describe('BitSet', () => { - it('cloneEmpty should return an empty set', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); - set1.add(1); - set1.add(3); - - let set2 = set1.cloneEmpty(); - - assertValues(set2, []); - }); - it('clone should return a set with the same values', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); @@ -42,7 +35,7 @@ describe('BitSet', () => { }); it('clear should remove all values from the set', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); @@ -52,7 +45,7 @@ describe('BitSet', () => { }); it('delete should remove values from the set', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); set1.add(5); @@ -63,11 +56,11 @@ describe('BitSet', () => { }); it('should intersect with another BitSet', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); - let set2 = set1.cloneEmpty(); + let set2 = new BitSet(5); set2.add(3); set2.add(5); @@ -76,11 +69,11 @@ describe('BitSet', () => { }); it('should union with another BitSet', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); - let set2 = set1.cloneEmpty(); + let set2 = new BitSet(5); set2.add(3); set2.add(5); @@ -89,11 +82,11 @@ describe('BitSet', () => { }); it('BitSet.union should create a new BitSet with the union', () => { - let set1 = BitSet.from([1, 2, 3, 4, 5]); + let set1 = new BitSet(5); set1.add(1); set1.add(3); - let set2 = set1.cloneEmpty(); + let set2 = new BitSet(5); set2.add(3); set2.add(5); @@ -102,18 +95,4 @@ describe('BitSet', () => { assertValues(set2, [3, 5]); assertValues(set3, [1, 3, 5]); }); - - it('returns an array of all values', () => { - let set = BitSet.from([1, 2, 3, 4]); - set.add(1); - set.add(3); - - assertValues(set, [3, 1]); - }); - - it('should return an error if a new item is added', () => { - let set = BitSet.from([1, 2, 3, 4]); - - assert.throws(() => set.add(5), /Item is missing from BitSet/); - }); }); diff --git a/packages/core/utils/src/BitSet.js b/packages/core/utils/src/BitSet.js deleted file mode 100644 index 71e2bce064f..00000000000 --- a/packages/core/utils/src/BitSet.js +++ /dev/null @@ -1,190 +0,0 @@ -// @flow strict-local -import nullthrows from 'nullthrows'; - -export class BitSet { - _value: RawBitSet; - _lookup: Map; - _items: Array; - - constructor({ - initial, - items, - lookup, - }: {| - items: Array, - lookup: Map, - initial?: BitSet | RawBitSet, - |}) { - if (initial instanceof BitSet) { - this._value = initial?._value; - } else if (initial) { - this._value = initial; - } else { - this._value = new RawBitSet(items.length); - } - - this._items = items; - this._lookup = lookup; - } - - static from(items: Array): BitSet { - let lookup: Map = new Map(); - for (let i = 0; i < items.length; i++) { - lookup.set(items[i], i); - } - - return new BitSet({items, lookup}); - } - - static union(a: BitSet, b: BitSet): BitSet { - let value = a._value.clone(); - value.union(b._value); - return new BitSet({ - initial: value, - lookup: a._lookup, - items: a._items, - }); - } - - #getIndex(item: Item) { - return nullthrows(this._lookup.get(item), 'Item is missing from BitSet'); - } - - add(item: Item) { - this._value.add(this.#getIndex(item)); - } - - delete(item: Item) { - this._value.delete(this.#getIndex(item)); - } - - has(item: Item): boolean { - return this._value.has(this.#getIndex(item)); - } - - intersect(v: BitSet) { - this._value.intersect(v._value); - } - - union(v: BitSet) { - this._value.union(v._value); - } - - unionRaw(v: RawBitSet) { - this._value.union(v); - } - - clear() { - this._value.clear(); - } - - cloneEmpty(): BitSet { - return new BitSet({ - lookup: this._lookup, - items: this._items, - }); - } - - clone(): BitSet { - return new BitSet({ - lookup: this._lookup, - items: this._items, - initial: this._value.clone(), - }); - } - - values(): Array { - let values = []; - this._value.forEach(i => { - values.push(this._items[i]); - }); - - return values; - } -} - -// Small wasm program that exposes the `ctz` instruction. -// https://developer.mozilla.org/en-US/docs/WebAssembly/Reference/Numeric/Count_trailing_zeros -const wasmBuf = new Uint8Array([ - 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, 0x01, - 0x7f, 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0d, 0x01, 0x09, 0x74, 0x72, - 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x30, 0x00, 0x00, 0x0a, 0x07, 0x01, 0x05, - 0x00, 0x20, 0x00, 0x68, 0x0b, 0x00, 0x0f, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x02, - 0x08, 0x01, 0x00, 0x01, 0x00, 0x03, 0x6e, 0x75, 0x6d, -]); - -// eslint-disable-next-line -const {trailing0} = new WebAssembly.Instance(new WebAssembly.Module(wasmBuf)) - .exports; - -export class RawBitSet { - bits: Uint32Array; - - constructor(maxBits: number) { - this.bits = new Uint32Array(Math.ceil(maxBits / 32)); - } - - clone(): RawBitSet { - let res = new RawBitSet(this.capacity); - res.bits.set(this.bits); - return res; - } - - get capacity(): number { - return this.bits.length * 32; - } - - add(bit: number) { - let i = bit >>> 5; - let b = bit & 31; - this.bits[i] |= 1 << b; - } - - delete(bit: number) { - let i = bit >>> 5; - let b = bit & 31; - this.bits[i] &= ~(1 << b); - } - - has(bit: number): boolean { - let i = bit >>> 5; - let b = bit & 31; - return Boolean(this.bits[i] & (1 << b)); - } - - clear() { - this.bits.fill(0); - } - - intersect(other: RawBitSet) { - for (let i = 0; i < this.bits.length; i++) { - this.bits[i] &= other.bits[i]; - } - } - - union(other: RawBitSet) { - for (let i = 0; i < this.bits.length; i++) { - this.bits[i] |= other.bits[i]; - } - } - - remove(other: RawBitSet) { - for (let i = 0; i < this.bits.length; i++) { - this.bits[i] &= ~other.bits[i]; - } - } - - forEach(fn: (bit: number) => void) { - // https://lemire.me/blog/2018/02/21/iterating-over-set-bits-quickly/ - let bits = this.bits; - for (let k = 0; k < bits.length; k++) { - let v = bits[k]; - while (v !== 0) { - let t = (v & -v) >>> 0; - // $FlowFixMe - fn((k << 5) + trailing0(v)); - v ^= t; - } - } - } -} diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index 179a4d226c4..ee499b0a3e6 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -85,5 +85,4 @@ export { loadSourceMap, remapSourceLocation, } from './sourcemap'; -export {BitSet, RawBitSet} from './BitSet'; export {default as stripAnsi} from 'strip-ansi';