From 360ee1931c7c1a79e12b76997593ff2ec204778f Mon Sep 17 00:00:00 2001 From: evanbacon Date: Wed, 22 Jun 2022 20:03:09 +0200 Subject: [PATCH] Add all require.context changes --- .../metro-runtime/src/polyfills/require.js | 11 ++++ .../metro/src/DeltaBundler/DeltaCalculator.js | 1 - .../metro/src/DeltaBundler/Worker.flow.js | 25 +++++++- .../metro/src/DeltaBundler/graphOperations.js | 61 +++++++++++++++++-- packages/metro/src/DeltaBundler/types.flow.js | 26 +++++++- packages/metro/src/IncrementalBundler.js | 13 ++++ .../ModuleGraph/worker/collectDependencies.js | 2 +- packages/metro/src/Server.js | 4 ++ packages/metro/src/lib/transformHelpers.js | 41 +++---------- 9 files changed, 142 insertions(+), 42 deletions(-) diff --git a/packages/metro-runtime/src/polyfills/require.js b/packages/metro-runtime/src/polyfills/require.js index aea905d05b..9c86f7633c 100644 --- a/packages/metro-runtime/src/polyfills/require.js +++ b/packages/metro-runtime/src/polyfills/require.js @@ -278,6 +278,17 @@ function metroImportAll(moduleId: ModuleID | VerboseModuleNameForDev | number) { } metroRequire.importAll = metroImportAll; +if (__DEV__) { + // The `require.context()` syntax is never executed in the runtime because it is converted + // to `require()` in `metro/src/ModuleGraph/worker/collectDependencies.js` after collecting + // dependencies. If the feature flag is not enabled then the conversion never takes place and this error is thrown (development only). + metroRequire.context = () => { + throw new Error( + `The experimental Metro feature \`require.context\` is not enabled in your project.\nThis can be enabled by setting the \`transformer.unstable_allowRequireContext\` property to \`true\` in the project \`metro.config.js\`.`, + ); + }; +} + let inGuard = false; function guardedLoadModule( moduleId: ModuleID, diff --git a/packages/metro/src/DeltaBundler/DeltaCalculator.js b/packages/metro/src/DeltaBundler/DeltaCalculator.js index 94cc28c442..6a9cbedf33 100644 --- a/packages/metro/src/DeltaBundler/DeltaCalculator.js +++ b/packages/metro/src/DeltaBundler/DeltaCalculator.js @@ -195,7 +195,6 @@ class DeltaCalculator extends EventEmitter { this._addedFiles.add(filePath); this._deletedFiles.delete(filePath); - this._modifiedFiles.delete(filePath); } else { this._modifiedFiles.add(filePath); diff --git a/packages/metro/src/DeltaBundler/Worker.flow.js b/packages/metro/src/DeltaBundler/Worker.flow.js index a8d0451406..633fe38deb 100644 --- a/packages/metro/src/DeltaBundler/Worker.flow.js +++ b/packages/metro/src/DeltaBundler/Worker.flow.js @@ -56,6 +56,30 @@ async function transform( transformOptions: JsTransformOptions, projectRoot: string, transformerConfig: TransformerConfig, + fileBuffer?: Buffer, +): Promise { + let data; + + if (fileBuffer && fileBuffer.type === 'Buffer') { + data = Buffer.from(fileBuffer.data); + } else { + data = fs.readFileSync(path.resolve(projectRoot, filename)); + } + return transformFile( + filename, + data, + transformOptions, + projectRoot, + transformerConfig, + ); +} + +async function transformFile( + filename: string, + data: Buffer, + transformOptions: JsTransformOptions, + projectRoot: string, + transformerConfig: TransformerConfig, ): Promise { // eslint-disable-next-line no-useless-call const Transformer = (require.call( @@ -71,7 +95,6 @@ async function transform( start_timestamp: process.hrtime(), }; - const data = fs.readFileSync(path.resolve(projectRoot, filename)); const sha1 = crypto.createHash('sha1').update(data).digest('hex'); const result = await Transformer.transform( diff --git a/packages/metro/src/DeltaBundler/graphOperations.js b/packages/metro/src/DeltaBundler/graphOperations.js index 530d7d62b1..814c899749 100644 --- a/packages/metro/src/DeltaBundler/graphOperations.js +++ b/packages/metro/src/DeltaBundler/graphOperations.js @@ -37,10 +37,16 @@ import type { Module, Options, TransformResultDependency, + RequireContext, } from './types.flow'; import CountingSet from '../lib/CountingSet'; +import { + appendContextQueryParam, + removeContextQueryParam, +} from '../lib/contextModule'; +import * as path from 'path'; const invariant = require('invariant'); const nullthrows = require('nullthrows'); @@ -115,11 +121,13 @@ type InternalOptions = $ReadOnly<{ onDependencyAdded: () => mixed, resolve: Options['resolve'], transform: Options['transform'], + transformContext: Options['transformContext'], shallow: boolean, }>; function getInternalOptions({ transform, + transformContext, resolve, onProgress, experimentalImportBundleSupport, @@ -131,6 +139,7 @@ function getInternalOptions({ return { experimentalImportBundleSupport, transform, + transformContext, resolve, onDependencyAdd: () => onProgress && onProgress(numProcessed, ++total), onDependencyAdded: () => onProgress && onProgress(++numProcessed, total), @@ -144,7 +153,7 @@ function getInternalOptions({ * dependency graph. * Instead of traversing the whole graph each time, it just calculates the * difference between runs by only traversing the added/removed dependencies. - * To do so, it uses the passed passed graph dependencies and it mutates it. + * To do so, it uses the passed graph dependencies and it mutates it. * The paths parameter contains the absolute paths of the root files that the * method should traverse. Normally, these paths should be the modified files * since the last traversal. @@ -255,7 +264,13 @@ async function traverseDependenciesForSingleFile( ): Promise { options.onDependencyAdd(); - await processModule(path, graph, delta, options); + await processModule( + path, + graph, + delta, + options, + graph.dependencies.get(path)?.contextParams, + ); options.onDependencyAdded(); } @@ -265,10 +280,20 @@ async function processModule( graph: Graph, delta: Delta, options: InternalOptions, + contextParams?: RequireContext, ): Promise> { + const resolvedContextParams = + contextParams || graph.dependencies.get(path)?.contextParams; + // Transform the file via the given option. // TODO: Unbind the transform method from options - const result = await options.transform(path); + let result; + if (resolvedContextParams) { + const modulePath = removeContextQueryParam(path); + result = await options.transformContext(modulePath, resolvedContextParams); + } else { + result = await options.transform(path); + } // Get the absolute path of all sub-dependencies (some of them could have been // moved but maintain the same relative path). @@ -288,6 +313,7 @@ async function processModule( // Update the module information. const module = { ...previousModule, + contextParams: resolvedContextParams, dependencies: new Map(previousDependencies), getSource: result.getSource, output: result.output, @@ -403,7 +429,13 @@ async function addDependency( delta.earlyInverseDependencies.set(path, new CountingSet()); options.onDependencyAdd(); - module = await processModule(path, graph, delta, options); + module = await processModule( + path, + graph, + delta, + options, + dependency.data.data.contextParams, + ); options.onDependencyAdded(); graph.dependencies.set(module.path, module); @@ -469,6 +501,27 @@ function resolveDependencies( const resolve = (parentPath: string, result: TransformResultDependency) => { const relativePath = result.name; try { + // `require.context` + if (result.data.contextParams) { + let absolutePath = path.join(parentPath, '..', result.name); + + // Ensure the filepath has uniqueness applied to ensure multiple `require.context` + // statements can be used to target the same file with different properties. + absolutePath = appendContextQueryParam( + absolutePath, + result.data.contextParams, + ); + + return [ + relativePath, + { + // TODO: Verify directory exists + // absolutePath: options.resolve(parentPath, dep.name), + absolutePath, + data: result, + }, + ]; + } return [ relativePath, { diff --git a/packages/metro/src/DeltaBundler/types.flow.js b/packages/metro/src/DeltaBundler/types.flow.js index f244607e12..4e3b065d37 100644 --- a/packages/metro/src/DeltaBundler/types.flow.js +++ b/packages/metro/src/DeltaBundler/types.flow.js @@ -10,12 +10,24 @@ 'use strict'; -import type {RequireContextParams} from '../ModuleGraph/worker/collectDependencies'; +import type { + RequireContextParams, + ContextMode, +} from '../ModuleGraph/worker/collectDependencies'; import type {PrivateState} from './graphOperations'; import type {JsTransformOptions} from 'metro-transform-worker'; import CountingSet from '../lib/CountingSet'; +export type RequireContext = { + /* Should search for files recursively. Optional, default `true` when `require.context` is used */ + recursive: boolean, + /* Filename filter pattern for use in `require.context`. Optional, default `.*` (any file) when `require.context` is used */ + filter: RegExp, + /** Mode for resolving dynamic dependencies. Defaults to `sync` */ + mode: ContextMode, +}; + export type MixedOutput = { +data: mixed, +type: string, @@ -63,6 +75,7 @@ export type Dependency = { }; export type Module = { + +contextParams?: RequireContext, +dependencies: Map, +inverseDependencies: CountingSet, +output: $ReadOnlyArray, @@ -104,6 +117,12 @@ export type TransformResultWithSource = $ReadOnly<{ getSource: () => Buffer, }>; +/** Transformer for generating `require.context` virtual module. */ +export type TransformContextFn = ( + string, + RequireContext, +) => Promise>; + export type TransformFn = string => Promise< TransformResultWithSource, >; @@ -115,11 +134,14 @@ export type AllowOptionalDependencies = | AllowOptionalDependenciesWithOptions; export type Options = { - +resolve: (from: string, to: string) => string, + +resolve: (from: string, to: string, context?: ?RequireContext) => string, +transform: TransformFn, + /** Given a path and require context, return a virtual context module. */ + +transformContext: TransformContextFn, +transformOptions: TransformInputOptions, +onProgress: ?(numProcessed: number, total: number) => mixed, +experimentalImportBundleSupport: boolean, + +unstable_allowRequireContext: boolean, +shallow: boolean, }; diff --git a/packages/metro/src/IncrementalBundler.js b/packages/metro/src/IncrementalBundler.js index d42c9d6cb6..240e3eea64 100644 --- a/packages/metro/src/IncrementalBundler.js +++ b/packages/metro/src/IncrementalBundler.js @@ -130,6 +130,8 @@ class IncrementalBundler { onProgress: otherOptions.onProgress, experimentalImportBundleSupport: this._config.transformer.experimentalImportBundleSupport, + unstable_allowRequireContext: + this._config.transformer.unstable_allowRequireContext, shallow: otherOptions.shallow, }); @@ -160,6 +162,13 @@ class IncrementalBundler { this._bundler, transformOptions.platform, ), + transformContext: await transformHelpers.getTransformContextFn( + absoluteEntryFiles, + this._bundler, + this._deltaBundler, + this._config, + transformOptions, + ), transform: await transformHelpers.getTransformFn( absoluteEntryFiles, this._bundler, @@ -171,6 +180,8 @@ class IncrementalBundler { onProgress: otherOptions.onProgress, experimentalImportBundleSupport: this._config.transformer.experimentalImportBundleSupport, + unstable_allowRequireContext: + this._config.transformer.unstable_allowRequireContext, shallow: otherOptions.shallow, }, ); @@ -225,6 +236,8 @@ class IncrementalBundler { shallow: otherOptions.shallow, experimentalImportBundleSupport: this._config.transformer.experimentalImportBundleSupport, + unstable_allowRequireContext: + this._config.transformer.unstable_allowRequireContext, }); const revisionId = createRevisionId(); const revisionPromise = (async () => { diff --git a/packages/metro/src/ModuleGraph/worker/collectDependencies.js b/packages/metro/src/ModuleGraph/worker/collectDependencies.js index e4a7d6e3b0..a67de8d055 100644 --- a/packages/metro/src/ModuleGraph/worker/collectDependencies.js +++ b/packages/metro/src/ModuleGraph/worker/collectDependencies.js @@ -38,7 +38,7 @@ export type Dependency = $ReadOnly<{ }>; // TODO: Convert to a Flow enum -type ContextMode = 'sync' | 'eager' | 'lazy' | 'lazy-once'; +export type ContextMode = 'sync' | 'eager' | 'lazy' | 'lazy-once'; type ContextFilter = {pattern: string, flags: string}; diff --git a/packages/metro/src/Server.js b/packages/metro/src/Server.js index 7edea0bd20..295ddf49a3 100644 --- a/packages/metro/src/Server.js +++ b/packages/metro/src/Server.js @@ -533,6 +533,8 @@ class Server { shallow: graphOptions.shallow, experimentalImportBundleSupport: this._config.transformer.experimentalImportBundleSupport, + unstable_allowRequireContext: + this._config.transformer.unstable_allowRequireContext, }); // For resources that support deletion, handle the DELETE method. @@ -1134,6 +1136,8 @@ class Server { shallow: graphOptions.shallow, experimentalImportBundleSupport: this._config.transformer.experimentalImportBundleSupport, + unstable_allowRequireContext: + this._config.transformer.unstable_allowRequireContext, }); let revision; const revPromise = this._bundler.getRevisionByGraphId(graphId); diff --git a/packages/metro/src/lib/transformHelpers.js b/packages/metro/src/lib/transformHelpers.js index 1ba2ccd9fe..d40d1ec328 100644 --- a/packages/metro/src/lib/transformHelpers.js +++ b/packages/metro/src/lib/transformHelpers.js @@ -139,43 +139,18 @@ async function getTransformContextFn( options, ); - // Cache all of the modules for intermittent updates. - const moduleCache = {}; - return async (modulePath: string, requireContext: RequireContext) => { const graph = await bundler.getDependencyGraph(); - let files = []; - if (modulePath in moduleCache && requireContext.delta) { - // Get the cached modules - files = moduleCache[modulePath]; - - // Remove files from the cache. - const deletedFiles = requireContext.delta.deletedFiles; - if (deletedFiles.size) { - files = files.filter(filePath => !deletedFiles.has(filePath)); - } + // TODO: Check delta changes to avoid having to look over all files each time + // this is a massive performance boost. - // Add files to the cache. - const addedFiles = requireContext.delta?.addedFiles; - addedFiles?.forEach(filePath => { - if ( - !files.includes(filePath) && - fileMatchesContext(modulePath, filePath, requireContext) - ) { - files.push(filePath); - } - }); - } else { - // Search against all files, this is very expensive. - // TODO: Maybe we could let the user specify which root to check against. - files = graph.matchFilesWithContext(modulePath, { - filter: requireContext.filter, - recursive: requireContext.recursive, - }); - } - - moduleCache[modulePath] = files; + // Search against all files, this is very expensive. + // TODO: Maybe we could let the user specify which root to check against. + const files = graph.matchFilesWithContext(modulePath, { + filter: requireContext.filter, + recursive: requireContext.recursive, + }); const template = getContextModuleTemplate( requireContext.mode,