From 984d6be2100da561736fb819315b774b55ec1ea2 Mon Sep 17 00:00:00 2001 From: aleclarson Date: Tue, 18 Sep 2018 12:19:52 -0400 Subject: [PATCH 1/4] feat: opaque symlink support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every "node_modules" directory within a project root is searched for linked dependencies. Each linked dependency can become a project root if its resolved path meets these criteria: - not inside a "node_modules" directory - not inside an existing project root Once all linked dependencies are found, collect every file (including symlinks) in each of the project roots. Two properties are added to the cache of "jest-haste-map": "links": Map of link paths (relative to rootDir) to their metadata "roots": Array of configured roots and linked roots One method is added to the "HasteFS" class: "follow": Lazily resolves any symlink found by the crawler ⚠️ The JS crawler does *not* yet support symlinks; only the Watchman crawler does. ⚠️ Watch mode does *not* yet support symlinks. --- packages/jest-haste-map/package.json | 1 + packages/jest-haste-map/src/HasteFS.ts | 30 +- .../src/__tests__/index.test.js | 5 +- .../__snapshots__/watchman.test.js.snap | 363 ++++++++++++++ .../src/crawlers/__tests__/watchman.test.js | 73 +-- packages/jest-haste-map/src/crawlers/node.ts | 1 + .../jest-haste-map/src/crawlers/watchman.ts | 457 +++++++++++------- packages/jest-haste-map/src/index.ts | 19 +- packages/jest-haste-map/src/types.ts | 8 + 9 files changed, 725 insertions(+), 232 deletions(-) create mode 100644 packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap diff --git a/packages/jest-haste-map/package.json b/packages/jest-haste-map/package.json index 26189086efc9..2a1d56031012 100644 --- a/packages/jest-haste-map/package.json +++ b/packages/jest-haste-map/package.json @@ -19,6 +19,7 @@ "jest-util": "^24.9.0", "jest-worker": "^24.9.0", "micromatch": "^3.1.10", + "realpath-native": "^1.0.0", "sane": "^4.0.3", "walker": "^1.0.7" }, diff --git a/packages/jest-haste-map/src/HasteFS.ts b/packages/jest-haste-map/src/HasteFS.ts index a15af1301dce..96d61278afb3 100644 --- a/packages/jest-haste-map/src/HasteFS.ts +++ b/packages/jest-haste-map/src/HasteFS.ts @@ -5,20 +5,32 @@ * LICENSE file in the root directory of this source tree. */ +import path from 'path'; import micromatch from 'micromatch'; +import {sync as realpath} from 'realpath-native'; import {replacePathSepForGlob} from 'jest-util'; import {Config} from '@jest/types'; -import {FileData} from './types'; +import {FileData, LinkData} from './types'; import * as fastPath from './lib/fast_path'; import H from './constants'; export default class HasteFS { private readonly _rootDir: Config.Path; private readonly _files: FileData; + private readonly _links: LinkData; - constructor({rootDir, files}: {rootDir: Config.Path; files: FileData}) { + constructor({ + rootDir, + files, + links, + }: { + rootDir: Config.Path; + files: FileData; + links: LinkData; + }) { this._rootDir = rootDir; this._files = files; + this._links = links; } getModuleName(file: Config.Path): string | null { @@ -52,6 +64,20 @@ export default class HasteFS { return this._getFileData(file) != null; } + follow(file: Config.Path): Config.Path { + const name = fastPath.relative(this._rootDir, file); + const link = this._links.get(name); + if (!link) { + return file; + } + if (!link[0]) { + const target = realpath(file); + link[0] = fastPath.relative(this._rootDir, target); + return target; + } + return path.join(this._rootDir, link[0]); + } + getAllFiles(): Array { return Array.from(this.getAbsoluteFileIterator()); } diff --git a/packages/jest-haste-map/src/__tests__/index.test.js b/packages/jest-haste-map/src/__tests__/index.test.js index 196094edd1c8..ad9bc12e90e1 100644 --- a/packages/jest-haste-map/src/__tests__/index.test.js +++ b/packages/jest-haste-map/src/__tests__/index.test.js @@ -102,6 +102,7 @@ jest.mock('graceful-fs', () => ({ error.code = 'ENOENT'; throw error; }), + realpath: jest.fn(path => path), writeFileSync: jest.fn((path, data, options) => { expect(options).toBe(require('v8').serialize ? undefined : 'utf8'); mockFs[path] = data; @@ -224,8 +225,6 @@ describe('HasteMap', () => { it('creates valid cache file paths', () => { jest.resetModuleRegistry(); - HasteMap = require('../'); - expect( HasteMap.getCacheFilePath('/', '@scoped/package', 'random-value'), ).toMatch(/^\/-scoped-package-(.*)$/); @@ -241,7 +240,6 @@ describe('HasteMap', () => { it('creates different cache file paths for different dependency extractor cache keys', () => { jest.resetModuleRegistry(); - const HasteMap = require('../'); const dependencyExtractor = require('./dependencyExtractor'); const config = { ...defaultConfig, @@ -256,7 +254,6 @@ describe('HasteMap', () => { it('creates different cache file paths for different hasteImplModulePath cache keys', () => { jest.resetModuleRegistry(); - const HasteMap = require('../'); const hasteImpl = require('./haste_impl'); hasteImpl.setCacheKey('foo'); const hasteMap1 = new HasteMap(defaultConfig); diff --git a/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap b/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap new file mode 100644 index 000000000000..78cba300046a --- /dev/null +++ b/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap @@ -0,0 +1,363 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`watchman watch does not add directory filters to query when watching a ROOT 1`] = ` +Array [ + Array [ + "watch-project", + "/root-mock/fruits", + ], + Array [ + "watch-project", + "/root-mock/vegetables", + ], + Array [ + "watch-project", + "/root-mock", + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "anyof", + Array [ + "type", + "l", + ], + Array [ + "allof", + Array [ + "type", + "f", + ], + Array [ + "anyof", + Array [ + "suffix", + "js", + ], + Array [ + "suffix", + "json", + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], +] +`; + +exports[`watchman watch returns a list of all files when there are no clocks 1`] = ` +Array [ + Array [ + "watch-project", + "/root-mock/fruits", + ], + Array [ + "watch-project", + "/root-mock/vegetables", + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "anyof", + Array [ + "type", + "l", + ], + Array [ + "allof", + Array [ + "type", + "f", + ], + Array [ + "anyof", + Array [ + "suffix", + "js", + ], + Array [ + "suffix", + "json", + ], + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], +] +`; diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index f72dc3fc5acb..4b42d08f7136 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -123,6 +123,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -134,34 +136,11 @@ describe('watchman watch', () => { expect(client.on).toBeCalled(); expect(client.on).toBeCalledWith('error', expect.any(Function)); - - // Call 0 and 1 are for ['watch-project'] - expect(calls[0][0][0]).toEqual('watch-project'); - expect(calls[1][0][0]).toEqual('watch-project'); - - // Call 2 is the query - const query = calls[2][0]; - expect(query[0]).toEqual('query'); - - expect(query[2].expression).toEqual([ - 'allof', - ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], - ['anyof', ['dirname', 'fruits'], ['dirname', 'vegetables']], - ]); - - expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); - - expect(query[2].glob).toEqual([ - 'fruits/**/*.js', - 'fruits/**/*.json', - 'vegetables/**/*.js', - 'vegetables/**/*.json', - ]); + expect(calls.map(call => call[0])).toMatchSnapshot(); expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }), ); @@ -203,6 +182,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json', 'zip'], ignore: pearMatcher, @@ -256,13 +237,15 @@ describe('watchman watch', () => { }; const clocks = createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }); return watchmanCrawl({ data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -274,7 +257,7 @@ describe('watchman watch', () => { expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:2', + '.': 'c:fake-clock:2', }), ); @@ -346,13 +329,15 @@ describe('watchman watch', () => { mockFiles.set(TOMATO_RELATIVE, mockTomatoMetadata); const clocks = createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }); return watchmanCrawl({ data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -364,7 +349,7 @@ describe('watchman watch', () => { expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:3', + '.': 'c:fake-clock:3', }), ); @@ -449,6 +434,8 @@ describe('watchman watch', () => { data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -514,6 +501,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -525,29 +514,11 @@ describe('watchman watch', () => { expect(client.on).toBeCalled(); expect(client.on).toBeCalledWith('error', expect.any(Function)); - - // First 3 calls are for ['watch-project'] - expect(calls[0][0][0]).toEqual('watch-project'); - expect(calls[1][0][0]).toEqual('watch-project'); - expect(calls[2][0][0]).toEqual('watch-project'); - - // Call 4 is the query - const query = calls[3][0]; - expect(query[0]).toEqual('query'); - - expect(query[2].expression).toEqual([ - 'allof', - ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], - ]); - - expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); - - expect(query[2].glob).toEqual(['**/*.js', '**/*.json']); + expect(calls.map(call => call[0])).toMatchSnapshot(); expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }), ); @@ -588,6 +559,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], rootDir: ROOT_MOCK, @@ -628,6 +601,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], rootDir: ROOT_MOCK, diff --git a/packages/jest-haste-map/src/crawlers/node.ts b/packages/jest-haste-map/src/crawlers/node.ts index 87bcb00b9680..8be9e93efb94 100644 --- a/packages/jest-haste-map/src/crawlers/node.ts +++ b/packages/jest-haste-map/src/crawlers/node.ts @@ -165,6 +165,7 @@ export = function nodeCrawl( removedFiles.delete(relativeFilePath); }); data.files = files; + data.links = new Map(); // TODO: support symlinks resolve({ hasteMap: data, diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 5e60ed59a7e4..153c5e9b9936 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -5,7 +5,9 @@ * LICENSE file in the root directory of this source tree. */ +import fs from 'fs'; import path from 'path'; +import {sync as realpath} from 'realpath-native'; import watchman from 'fb-watchman'; import {Config} from '@jest/types'; import * as fastPath from '../lib/fast_path'; @@ -16,6 +18,7 @@ import { CrawlerOptions, FileMetaData, FileData, + LinkMetaData, } from '../types'; type WatchmanRoots = Map>; @@ -23,6 +26,23 @@ type WatchmanRoots = Map>; const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html'; +// Matches symlinks in "node_modules" directories. +const nodeModulesExpression = [ + 'allof', + ['type', 'l'], + [ + 'anyof', + ...['**/node_modules/*', '**/node_modules/@*/*'].map(glob => [ + 'match', + glob, + 'wholename', + {includedotfiles: true}, + ]), + ], +]; + +const NODE_MODULES = path.sep + 'node_modules' + path.sep; + function WatchmanError(error: Error): Error { error.message = `Watchman error: ${error.message.trim()}. Make sure watchman ` + @@ -37,14 +57,7 @@ export = async function watchmanCrawl( removedFiles: FileData; hasteMap: InternalHasteMap; }> { - const fields = ['name', 'exists', 'mtime_ms', 'size']; - const {data, extensions, ignore, rootDir, roots} = options; - const defaultWatchExpression = [ - 'allof', - ['type', 'f'], - ['anyof', ...extensions.map(extension => ['suffix', extension])], - ]; - const clocks = data.clocks; + const {data, extensions, ignore, rootDir} = options; const client = new watchman.Client(); let clientError; @@ -58,208 +71,312 @@ export = async function watchmanCrawl( ), ); + type WatchmanFile = { + name: string; + exists: boolean; + mtime_ms: number; + size: number; + type: string; + 'content.sha1hex'?: string; + }; + + const fields = ['name', 'exists', 'mtime_ms', 'size', 'type']; if (options.computeSha1) { const {capabilities} = await cmd('list-capabilities'); - - if (capabilities.indexOf('field-content.sha1hex') !== -1) { + if (capabilities.includes('field-content.sha1hex')) { fields.push('content.sha1hex'); } } - async function getWatchmanRoots( - roots: Array, - ): Promise { - const watchmanRoots = new Map(); - await Promise.all( - roots.map(async root => { - const response = await cmd('watch-project', root); - const existing = watchmanRoots.get(response.watch); - // A root can only be filtered if it was never seen with a - // relative_path before. - const canBeFiltered = !existing || existing.length > 0; - - if (canBeFiltered) { - if (response.relative_path) { - watchmanRoots.set( - response.watch, - (existing || []).concat(response.relative_path), - ); - } else { - // Make the filter directories an empty array to signal that this - // root was already seen and needs to be watched for all files or - // directories. - watchmanRoots.set(response.watch, []); - } + // Clone the clockspec cache to avoid mutations during the watch phase. + const clocks = new Map(data.clocks); + + /** + * Fetch an array of files that match the given expression and are contained + * by the given `watchRoot` (with directory filters applied). + * + * When the `watchRoot` has a cached Watchman clockspec, only changed files + * are returned. The cloned clockspec cache is updated on every query. + * + * The given `watchRoot` must be absolute. + */ + const queryRequest = async ( + watchRoot: Config.Path, + dirs: Array, + expression: Array, + ) => { + if (dirs.length) { + expression = [ + 'allof', + ['anyof', ...dirs.map(dir => ['dirname', dir])], + expression, + ]; + } + + const relativeRoot = fastPath.relative(rootDir, watchRoot) || '.'; + const response = await cmd('query', watchRoot, { + expression, + fields, + // Use the cached clockspec since `queryRequest` is called twice per root. + since: data.clocks.get(relativeRoot), + }); + + if ('warning' in response) { + console.warn('watchman warning:', response.warning); + } + + clocks.set(relativeRoot, response.clock); + return response; + }; + + // The cached array of project roots. + const roots: Config.Path[] = []; + const isValidRoot = (root: Config.Path) => + !root.includes(NODE_MODULES) && + !roots.includes(root) && + !roots.find(root => root.startsWith(root + path.sep)); + + // Ensure every configured root is valid. + options.roots + .map(root => realpath(root)) + .sort((a, b) => a.length - b.length) + .forEach(root => { + if (isValidRoot(root)) { + roots.push(root); + } + }); + + // Ensure every linked root is searched if valid. + data.roots.forEach(root => { + root = path.resolve(rootDir, root); + if (isValidRoot(root)) { + try { + if (fs.statSync(root).isDirectory()) { + roots.push(root); } - }), + } catch (e) {} + } + }); + + // Track which directories share a "watch root" (a common ancestor identified + // by Watchman). Use an empty array to denote a project root that is its own + // watch root. These arrays are used as "directory filters" in the 2nd phase. + const watched = new Map(); + + /** + * Look for linked dependencies in every "node_modules" directory within the + * given root. Repeat for every linked dependency found that resolves to a + * directory which exists outside of all known project roots. + */ + const findDependencyLinks = async (root: Config.Path) => { + const response = await cmd('watch-project', root); + const watchRoot = normalizePathSep(response.watch); + + let dirs = watched.get(watchRoot); + if (!dirs) { + watched.set(watchRoot, (dirs = [])); + } else if (!dirs.length) { + return; // Ensure no directory filters are used. + } + + const dir = normalizePathSep(response.relative_path || ''); + if (dir) { + if (dirs.includes(dir)) { + return; // Avoid crawling the same directory twice. + } + dirs.push(dir); + } else { + // Ensure no directory filters are used. + dirs.length = 0; + } + + // Search every "node_modules" directory in the entire tree. + // It should be faster for Watchman to do this in one fell swoop. + const queryResponse = await queryRequest( + watchRoot, + dir ? [dir] : [], + nodeModulesExpression, ); - return watchmanRoots; - } - async function queryWatchmanForDirs(rootProjectDirMappings: WatchmanRoots) { - const files = new Map(); - let isFresh = false; await Promise.all( - Array.from(rootProjectDirMappings).map( - async ([root, directoryFilters]) => { - const expression = Array.from(defaultWatchExpression); - const glob = []; - - if (directoryFilters.length > 0) { - expression.push([ - 'anyof', - ...directoryFilters.map(dir => ['dirname', dir]), - ]); - - for (const directory of directoryFilters) { - for (const extension of extensions) { - glob.push(`${directory}/**/*.${extension}`); - } - } - } else { - for (const extension of extensions) { - glob.push(`**/*.${extension}`); - } - } + queryResponse.files.map(async (link: WatchmanFile) => { + const name = normalizePathSep(link.name); + const linkPath = path.join(watchRoot, name); - const relativeRoot = fastPath.relative(rootDir, root); - const query = clocks.has(relativeRoot) - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks.get(relativeRoot)} - : // Otherwise use the `glob` filter - {expression, fields, glob}; + if (!link.exists || ignore(linkPath)) { + data.links.delete(linkPath); + return; + } - const response = await cmd('query', root, query); + let target; + try { + target = realpath(linkPath); + } catch (e) { + return; // Skip broken symlinks. + } - if ('warning' in response) { - console.warn('watchman warning: ', response.warning); - } + const metaData = data.links.get(linkPath); + const mtime = testModified(metaData, link.mtime_ms); + if (mtime !== 0) { + // See ../constants.ts + data.links.set(linkPath, [target, mtime]); + } - isFresh = isFresh || response.is_fresh_instance; - files.set(root, response); - }, - ), + if (fs.statSync(target).isDirectory() && isValidRoot(target)) { + roots.push(target); + await findDependencyLinks(target); + } + }), ); + }; - return { - files, - isFresh, - }; - } - - let files = data.files; + let isFresh = false; let removedFiles = new Map(); const changedFiles = new Map(); - let watchmanFiles: Map; - let isFresh = false; try { - const watchmanRoots = await getWatchmanRoots(roots); - const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); + await Promise.all(roots.map(findDependencyLinks)); + + const crawlExpression = [ + 'anyof', + ['type', 'l'], + [ + 'allof', + ['type', 'f'], + ['anyof', ...extensions.map(extension => ['suffix', extension])], + ], + ]; + + const watchRoots = Array.from(watched.keys()); + const crawled = await Promise.all( + watchRoots.map(async watchRoot => { + const queryResponse = await queryRequest( + watchRoot, + watched.get(watchRoot)!, + crawlExpression, + ); + if (!isFresh) { + isFresh = queryResponse.is_fresh_instance; + } + return queryResponse.files; + }), + ); - // Reset the file map if watchman was restarted and sends us a list of - // files. - if (watchmanFileResults.isFresh) { - files = new Map(); + // Reset the file map if Watchman refreshed. + if (isFresh) { removedFiles = new Map(data.files); - isFresh = true; + data.files = new Map(); + data.links = new Map(); } - watchmanFiles = watchmanFileResults.files; - } finally { - client.end(); - } - - if (clientError) { - throw clientError; - } - - // TODO: remove non-null - for (const [watchRoot, response] of watchmanFiles!) { - const fsRoot = normalizePathSep(watchRoot); - const relativeFsRoot = fastPath.relative(rootDir, fsRoot); - clocks.set(relativeFsRoot, response.clock); - - for (const fileData of response.files) { - const filePath = fsRoot + path.sep + normalizePathSep(fileData.name); - const relativeFilePath = fastPath.relative(rootDir, filePath); - const existingFileData = data.files.get(relativeFilePath); - - // If watchman is fresh, the removed files map starts with all files - // and we remove them as we verify they still exist. - if (isFresh && existingFileData && fileData.exists) { - removedFiles.delete(relativeFilePath); - } + // Update the file map and symlink map. + crawled.forEach((files: WatchmanFile[], i) => { + const watchRoot = watchRoots[i]; + for (const file of files) { + const name = normalizePathSep(file.name); + const filePath = path.join(watchRoot, name); + const relativeFilePath = fastPath.relative(rootDir, filePath); + const isLinked = file.type === 'l'; + + const existingFiles: Map = isLinked + ? data.links + : data.files; + + const existingFile = existingFiles.get(relativeFilePath); + if (isFresh && existingFile && file.exists) { + // If watchman is fresh, the removed files map starts with all files + // and we remove them as we verify they still exist. + removedFiles.delete(relativeFilePath); + } - if (!fileData.exists) { - // No need to act on files that do not exist and were not tracked. - if (existingFileData) { - files.delete(relativeFilePath); + if (!file.exists || ignore(filePath)) { + // No need to act on files that do not exist and were not tracked. + if (existingFile) { + existingFiles.delete(relativeFilePath); - // If watchman is not fresh, we will know what specific files were - // deleted since we last ran and can track only those files. - if (!isFresh) { - removedFiles.set(relativeFilePath, existingFileData); + // If watchman is not fresh, we will know what specific files were + // deleted since we last ran and can track only those files. + if (!isFresh) { + removedFiles.set(relativeFilePath, existingFile); + } } + continue; } - } else if (!ignore(filePath)) { - const mtime = - typeof fileData.mtime_ms === 'number' - ? fileData.mtime_ms - : fileData.mtime_ms.toNumber(); - const size = fileData.size; - - let sha1hex = fileData['content.sha1hex']; - if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { + + let sha1hex = file['content.sha1hex'] || null; + if (sha1hex && sha1hex.length !== 40) { sha1hex = null; } - let nextData: FileMetaData; - - if (existingFileData && existingFileData[H.MTIME] === mtime) { - nextData = existingFileData; - } else if ( - existingFileData && - sha1hex && - existingFileData[H.SHA1] === sha1hex - ) { - nextData = [ - existingFileData[0], - mtime, - existingFileData[2], - existingFileData[3], - existingFileData[4], - existingFileData[5], - ]; - } else { - // See ../constants.ts - nextData = ['', mtime, size, 0, '', sha1hex]; - } + const mtime = testModified(existingFile, file.mtime_ms); + if (mtime !== 0) { + if (isLinked) { + // See ../constants.ts + existingFiles.set(relativeFilePath, [undefined, mtime]); + continue; + } + + let nextData: FileMetaData; + if (sha1hex && existingFile && existingFile[H.SHA1] === sha1hex) { + nextData = [ + existingFile[0], + mtime, + existingFile[2], + existingFile[3], + existingFile[4], + existingFile[5], + ]; + } else { + // See ../constants.ts + nextData = ['', mtime, file.size, 0, '', sha1hex]; + } - const mappings = options.mapper ? options.mapper(filePath) : null; - - if (mappings) { - for (const absoluteVirtualFilePath of mappings) { - if (!ignore(absoluteVirtualFilePath)) { - const relativeVirtualFilePath = fastPath.relative( - rootDir, - absoluteVirtualFilePath, - ); - files.set(relativeVirtualFilePath, nextData); - changedFiles.set(relativeVirtualFilePath, nextData); + const mappings = options.mapper ? options.mapper(filePath) : null; + + if (mappings) { + for (const absoluteVirtualFilePath of mappings) { + if (!ignore(absoluteVirtualFilePath)) { + const relativeVirtualFilePath = fastPath.relative( + rootDir, + absoluteVirtualFilePath, + ); + data.files.set(relativeVirtualFilePath, nextData); + changedFiles.set(relativeVirtualFilePath, nextData); + } } + } else { + data.files.set(relativeFilePath, nextData); + changedFiles.set(relativeFilePath, nextData); } - } else { - files.set(relativeFilePath, nextData); - changedFiles.set(relativeFilePath, nextData); } } - } + }); + } finally { + client.end(); + } + if (clientError) { + throw clientError; } - data.files = files; + data.roots = roots.map(root => fastPath.relative(rootDir, root)); + data.clocks = clocks; return { changedFiles: isFresh ? undefined : changedFiles, hasteMap: data, removedFiles, }; }; + +/** + * Check if the file data has been modified since last cached. + * + * Returns the 2nd argument if modified, else zero. + */ +function testModified( + metaData: FileMetaData | LinkMetaData | undefined, + mtime: number | {toNumber(): number}, +) { + if (typeof mtime !== 'number') { + mtime = mtime.toNumber(); + } + return !metaData || metaData[H.MTIME] !== mtime ? mtime : 0; +} diff --git a/packages/jest-haste-map/src/index.ts b/packages/jest-haste-map/src/index.ts index 4a14bebc2eee..c8e634ea8e25 100644 --- a/packages/jest-haste-map/src/index.ts +++ b/packages/jest-haste-map/src/index.ts @@ -176,6 +176,8 @@ const getWhiteList = (list: Array | undefined): RegExp | null => { * type HasteMap = { * clocks: WatchmanClocks, * files: {[filepath: string]: FileMetaData}, + * links: {[filepath: string]: LinkMetaData}, + * roots: string[], * map: {[id: string]: ModuleMapItem}, * mocks: {[id: string]: string}, * } @@ -361,6 +363,7 @@ class HasteMap extends EventEmitter { const rootDir = this._options.rootDir; const hasteFS = new HasteFS({ files: hasteMap.files, + links: hasteMap.links, rootDir, }); const moduleMap = new HasteModuleMap({ @@ -817,8 +820,7 @@ class HasteMap extends EventEmitter { let mustCopy = true; const createWatcher = (root: Config.Path): Promise => { - // @ts-ignore: TODO how? "Cannot use 'new' with an expression whose type lacks a call or construct signature." - const watcher = new Watcher(root, { + const watcher = new Watcher(path.resolve(rootDir, root), { dot: false, glob: extensions.map(extension => '**/*.' + extension), ignored: ignorePattern, @@ -845,6 +847,7 @@ class HasteMap extends EventEmitter { eventsQueue, hasteFS: new HasteFS({ files: hasteMap.files, + links: hasteMap.links, rootDir, }), moduleMap: new HasteModuleMap({ @@ -897,6 +900,8 @@ class HasteMap extends EventEmitter { clocks: new Map(hasteMap.clocks), duplicates: new Map(hasteMap.duplicates), files: new Map(hasteMap.files), + links: new Map(hasteMap.links), + roots: [...hasteMap.roots], map: new Map(hasteMap.map), mocks: new Map(hasteMap.mocks), }; @@ -987,11 +992,9 @@ class HasteMap extends EventEmitter { }; this._changeInterval = setInterval(emitChange, CHANGE_INTERVAL); - return Promise.all(this._options.roots.map(createWatcher)).then( - watchers => { - this._watchers = watchers; - }, - ); + return Promise.all(hasteMap.roots.map(createWatcher)).then(watchers => { + this._watchers = watchers; + }); } /** @@ -1109,6 +1112,8 @@ class HasteMap extends EventEmitter { clocks: new Map(), duplicates: new Map(), files: new Map(), + links: new Map(), + roots: [], map: new Map(), mocks: new Map(), }; diff --git a/packages/jest-haste-map/src/types.ts b/packages/jest-haste-map/src/types.ts index 9f776911a225..a6edeab31ede 100644 --- a/packages/jest-haste-map/src/types.ts +++ b/packages/jest-haste-map/src/types.ts @@ -45,6 +45,7 @@ export type HasteImpl = { }; export type FileData = Map; +export type LinkData = Map; export type FileMetaData = [ /* id */ string, @@ -55,6 +56,11 @@ export type FileMetaData = [ /* sha1 */ string | null | undefined, ]; +export type LinkMetaData = [ + /* target */ string | undefined, + /* mtime */ number +]; + export type MockData = Map; export type ModuleMapData = Map; export type WatchmanClocks = Map; @@ -67,6 +73,8 @@ export type InternalHasteMap = { clocks: WatchmanClocks; duplicates: DuplicatesIndex; files: FileData; + links: LinkData; + roots: Config.Path[]; map: ModuleMapData; mocks: MockData; }; From b821a191c7a4f9f7c0039e7bde765fbdeea1bf89 Mon Sep 17 00:00:00 2001 From: Alec Larson Date: Fri, 8 Mar 2019 16:44:02 -0500 Subject: [PATCH 2/4] fix: dependency links not being saved in cache --- .../jest-haste-map/src/crawlers/watchman.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 153c5e9b9936..83a68dc8872b 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -163,6 +163,10 @@ export = async function watchmanCrawl( // watch root. These arrays are used as "directory filters" in the 2nd phase. const watched = new Map(); + // Store newfound dependency links in a local variable until the crawl phase, + // which is when we can know if Watchman is a fresh instance. + const dependencyLinks = new Map(); + /** * Look for linked dependencies in every "node_modules" directory within the * given root. Repeat for every linked dependency found that resolves to a @@ -217,10 +221,10 @@ export = async function watchmanCrawl( const metaData = data.links.get(linkPath); const mtime = testModified(metaData, link.mtime_ms); - if (mtime !== 0) { - // See ../constants.ts - data.links.set(linkPath, [target, mtime]); - } + dependencyLinks.set( + linkPath, + mtime !== 0 ? [target, mtime] : metaData!, + ); if (fs.statSync(target).isDirectory() && isValidRoot(target)) { roots.push(target); @@ -265,7 +269,11 @@ export = async function watchmanCrawl( if (isFresh) { removedFiles = new Map(data.files); data.files = new Map(); - data.links = new Map(); + data.links = dependencyLinks; + } else { + dependencyLinks.forEach((link, linkPath) => { + data.links.set(linkPath, link); + }); } // Update the file map and symlink map. From 58384db8a4948e671b6271c6da1b62224a037062 Mon Sep 17 00:00:00 2001 From: Alec Larson Date: Fri, 8 Mar 2019 17:01:06 -0500 Subject: [PATCH 3/4] fix: use relative path as key in "links" map --- packages/jest-haste-map/src/crawlers/watchman.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 83a68dc8872b..20dd7db3965e 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -206,9 +206,10 @@ export = async function watchmanCrawl( queryResponse.files.map(async (link: WatchmanFile) => { const name = normalizePathSep(link.name); const linkPath = path.join(watchRoot, name); + const relativePath = fastPath.relative(rootDir, linkPath); if (!link.exists || ignore(linkPath)) { - data.links.delete(linkPath); + data.links.delete(relativePath); return; } @@ -219,10 +220,10 @@ export = async function watchmanCrawl( return; // Skip broken symlinks. } - const metaData = data.links.get(linkPath); + const metaData = data.links.get(relativePath); const mtime = testModified(metaData, link.mtime_ms); dependencyLinks.set( - linkPath, + relativePath, mtime !== 0 ? [target, mtime] : metaData!, ); From 377ecb362e5cadbddd2911820b77b689a33d8c69 Mon Sep 17 00:00:00 2001 From: Alec Larson Date: Fri, 8 Mar 2019 17:28:56 -0500 Subject: [PATCH 4/4] fix: forgot to make the target relative --- packages/jest-haste-map/src/crawlers/watchman.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 20dd7db3965e..3bf05f17f1a4 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -224,7 +224,7 @@ export = async function watchmanCrawl( const mtime = testModified(metaData, link.mtime_ms); dependencyLinks.set( relativePath, - mtime !== 0 ? [target, mtime] : metaData!, + mtime !== 0 ? [fastPath.relative(rootDir, target), mtime] : metaData!, ); if (fs.statSync(target).isDirectory() && isValidRoot(target)) {