diff --git a/packages/jest-haste-map/package.json b/packages/jest-haste-map/package.json index 26189086efc9..2a1d56031012 100644 --- a/packages/jest-haste-map/package.json +++ b/packages/jest-haste-map/package.json @@ -19,6 +19,7 @@ "jest-util": "^24.9.0", "jest-worker": "^24.9.0", "micromatch": "^3.1.10", + "realpath-native": "^1.0.0", "sane": "^4.0.3", "walker": "^1.0.7" }, diff --git a/packages/jest-haste-map/src/HasteFS.ts b/packages/jest-haste-map/src/HasteFS.ts index a15af1301dce..96d61278afb3 100644 --- a/packages/jest-haste-map/src/HasteFS.ts +++ b/packages/jest-haste-map/src/HasteFS.ts @@ -5,20 +5,32 @@ * LICENSE file in the root directory of this source tree. */ +import path from 'path'; import micromatch from 'micromatch'; +import {sync as realpath} from 'realpath-native'; import {replacePathSepForGlob} from 'jest-util'; import {Config} from '@jest/types'; -import {FileData} from './types'; +import {FileData, LinkData} from './types'; import * as fastPath from './lib/fast_path'; import H from './constants'; export default class HasteFS { private readonly _rootDir: Config.Path; private readonly _files: FileData; + private readonly _links: LinkData; - constructor({rootDir, files}: {rootDir: Config.Path; files: FileData}) { + constructor({ + rootDir, + files, + links, + }: { + rootDir: Config.Path; + files: FileData; + links: LinkData; + }) { this._rootDir = rootDir; this._files = files; + this._links = links; } getModuleName(file: Config.Path): string | null { @@ -52,6 +64,20 @@ export default class HasteFS { return this._getFileData(file) != null; } + follow(file: Config.Path): Config.Path { + const name = fastPath.relative(this._rootDir, file); + const link = this._links.get(name); + if (!link) { + return file; + } + if (!link[0]) { + const target = realpath(file); + link[0] = fastPath.relative(this._rootDir, target); + return target; + } + return path.join(this._rootDir, link[0]); + } + getAllFiles(): Array { return Array.from(this.getAbsoluteFileIterator()); } diff --git a/packages/jest-haste-map/src/__tests__/index.test.js b/packages/jest-haste-map/src/__tests__/index.test.js index 196094edd1c8..ad9bc12e90e1 100644 --- a/packages/jest-haste-map/src/__tests__/index.test.js +++ b/packages/jest-haste-map/src/__tests__/index.test.js @@ -102,6 +102,7 @@ jest.mock('graceful-fs', () => ({ error.code = 'ENOENT'; throw error; }), + realpath: jest.fn(path => path), writeFileSync: jest.fn((path, data, options) => { expect(options).toBe(require('v8').serialize ? undefined : 'utf8'); mockFs[path] = data; @@ -224,8 +225,6 @@ describe('HasteMap', () => { it('creates valid cache file paths', () => { jest.resetModuleRegistry(); - HasteMap = require('../'); - expect( HasteMap.getCacheFilePath('/', '@scoped/package', 'random-value'), ).toMatch(/^\/-scoped-package-(.*)$/); @@ -241,7 +240,6 @@ describe('HasteMap', () => { it('creates different cache file paths for different dependency extractor cache keys', () => { jest.resetModuleRegistry(); - const HasteMap = require('../'); const dependencyExtractor = require('./dependencyExtractor'); const config = { ...defaultConfig, @@ -256,7 +254,6 @@ describe('HasteMap', () => { it('creates different cache file paths for different hasteImplModulePath cache keys', () => { jest.resetModuleRegistry(); - const HasteMap = require('../'); const hasteImpl = require('./haste_impl'); hasteImpl.setCacheKey('foo'); const hasteMap1 = new HasteMap(defaultConfig); diff --git a/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap b/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap new file mode 100644 index 000000000000..78cba300046a --- /dev/null +++ b/packages/jest-haste-map/src/crawlers/__tests__/__snapshots__/watchman.test.js.snap @@ -0,0 +1,363 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`watchman watch does not add directory filters to query when watching a ROOT 1`] = ` +Array [ + Array [ + "watch-project", + "/root-mock/fruits", + ], + Array [ + "watch-project", + "/root-mock/vegetables", + ], + Array [ + "watch-project", + "/root-mock", + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "anyof", + Array [ + "type", + "l", + ], + Array [ + "allof", + Array [ + "type", + "f", + ], + Array [ + "anyof", + Array [ + "suffix", + "js", + ], + Array [ + "suffix", + "json", + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], +] +`; + +exports[`watchman watch returns a list of all files when there are no clocks 1`] = ` +Array [ + Array [ + "watch-project", + "/root-mock/fruits", + ], + Array [ + "watch-project", + "/root-mock/vegetables", + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "allof", + Array [ + "type", + "l", + ], + Array [ + "anyof", + Array [ + "match", + "**/node_modules/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + Array [ + "match", + "**/node_modules/@*/*", + "wholename", + Object { + "includedotfiles": true, + }, + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], + Array [ + "query", + "/root-mock", + Object { + "expression": Array [ + "allof", + Array [ + "anyof", + Array [ + "dirname", + "fruits", + ], + Array [ + "dirname", + "vegetables", + ], + ], + Array [ + "anyof", + Array [ + "type", + "l", + ], + Array [ + "allof", + Array [ + "type", + "f", + ], + Array [ + "anyof", + Array [ + "suffix", + "js", + ], + Array [ + "suffix", + "json", + ], + ], + ], + ], + ], + "fields": Array [ + "name", + "exists", + "mtime_ms", + "size", + "type", + ], + "since": undefined, + }, + ], +] +`; diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index f72dc3fc5acb..4b42d08f7136 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -123,6 +123,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -134,34 +136,11 @@ describe('watchman watch', () => { expect(client.on).toBeCalled(); expect(client.on).toBeCalledWith('error', expect.any(Function)); - - // Call 0 and 1 are for ['watch-project'] - expect(calls[0][0][0]).toEqual('watch-project'); - expect(calls[1][0][0]).toEqual('watch-project'); - - // Call 2 is the query - const query = calls[2][0]; - expect(query[0]).toEqual('query'); - - expect(query[2].expression).toEqual([ - 'allof', - ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], - ['anyof', ['dirname', 'fruits'], ['dirname', 'vegetables']], - ]); - - expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); - - expect(query[2].glob).toEqual([ - 'fruits/**/*.js', - 'fruits/**/*.json', - 'vegetables/**/*.js', - 'vegetables/**/*.json', - ]); + expect(calls.map(call => call[0])).toMatchSnapshot(); expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }), ); @@ -203,6 +182,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json', 'zip'], ignore: pearMatcher, @@ -256,13 +237,15 @@ describe('watchman watch', () => { }; const clocks = createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }); return watchmanCrawl({ data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -274,7 +257,7 @@ describe('watchman watch', () => { expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:2', + '.': 'c:fake-clock:2', }), ); @@ -346,13 +329,15 @@ describe('watchman watch', () => { mockFiles.set(TOMATO_RELATIVE, mockTomatoMetadata); const clocks = createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }); return watchmanCrawl({ data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -364,7 +349,7 @@ describe('watchman watch', () => { expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:3', + '.': 'c:fake-clock:3', }), ); @@ -449,6 +434,8 @@ describe('watchman watch', () => { data: { clocks, files: mockFiles, + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -514,6 +501,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], ignore: pearMatcher, @@ -525,29 +514,11 @@ describe('watchman watch', () => { expect(client.on).toBeCalled(); expect(client.on).toBeCalledWith('error', expect.any(Function)); - - // First 3 calls are for ['watch-project'] - expect(calls[0][0][0]).toEqual('watch-project'); - expect(calls[1][0][0]).toEqual('watch-project'); - expect(calls[2][0][0]).toEqual('watch-project'); - - // Call 4 is the query - const query = calls[3][0]; - expect(query[0]).toEqual('query'); - - expect(query[2].expression).toEqual([ - 'allof', - ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], - ]); - - expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']); - - expect(query[2].glob).toEqual(['**/*.js', '**/*.json']); + expect(calls.map(call => call[0])).toMatchSnapshot(); expect(hasteMap.clocks).toEqual( createMap({ - '': 'c:fake-clock:1', + '.': 'c:fake-clock:1', }), ); @@ -588,6 +559,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], rootDir: ROOT_MOCK, @@ -628,6 +601,8 @@ describe('watchman watch', () => { data: { clocks: new Map(), files: new Map(), + links: new Map(), + roots: [], }, extensions: ['js', 'json'], rootDir: ROOT_MOCK, diff --git a/packages/jest-haste-map/src/crawlers/node.ts b/packages/jest-haste-map/src/crawlers/node.ts index 87bcb00b9680..8be9e93efb94 100644 --- a/packages/jest-haste-map/src/crawlers/node.ts +++ b/packages/jest-haste-map/src/crawlers/node.ts @@ -165,6 +165,7 @@ export = function nodeCrawl( removedFiles.delete(relativeFilePath); }); data.files = files; + data.links = new Map(); // TODO: support symlinks resolve({ hasteMap: data, diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 5e60ed59a7e4..3bf05f17f1a4 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -5,7 +5,9 @@ * LICENSE file in the root directory of this source tree. */ +import fs from 'fs'; import path from 'path'; +import {sync as realpath} from 'realpath-native'; import watchman from 'fb-watchman'; import {Config} from '@jest/types'; import * as fastPath from '../lib/fast_path'; @@ -16,6 +18,7 @@ import { CrawlerOptions, FileMetaData, FileData, + LinkMetaData, } from '../types'; type WatchmanRoots = Map>; @@ -23,6 +26,23 @@ type WatchmanRoots = Map>; const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting.html'; +// Matches symlinks in "node_modules" directories. +const nodeModulesExpression = [ + 'allof', + ['type', 'l'], + [ + 'anyof', + ...['**/node_modules/*', '**/node_modules/@*/*'].map(glob => [ + 'match', + glob, + 'wholename', + {includedotfiles: true}, + ]), + ], +]; + +const NODE_MODULES = path.sep + 'node_modules' + path.sep; + function WatchmanError(error: Error): Error { error.message = `Watchman error: ${error.message.trim()}. Make sure watchman ` + @@ -37,14 +57,7 @@ export = async function watchmanCrawl( removedFiles: FileData; hasteMap: InternalHasteMap; }> { - const fields = ['name', 'exists', 'mtime_ms', 'size']; - const {data, extensions, ignore, rootDir, roots} = options; - const defaultWatchExpression = [ - 'allof', - ['type', 'f'], - ['anyof', ...extensions.map(extension => ['suffix', extension])], - ]; - const clocks = data.clocks; + const {data, extensions, ignore, rootDir} = options; const client = new watchman.Client(); let clientError; @@ -58,208 +71,321 @@ export = async function watchmanCrawl( ), ); + type WatchmanFile = { + name: string; + exists: boolean; + mtime_ms: number; + size: number; + type: string; + 'content.sha1hex'?: string; + }; + + const fields = ['name', 'exists', 'mtime_ms', 'size', 'type']; if (options.computeSha1) { const {capabilities} = await cmd('list-capabilities'); - - if (capabilities.indexOf('field-content.sha1hex') !== -1) { + if (capabilities.includes('field-content.sha1hex')) { fields.push('content.sha1hex'); } } - async function getWatchmanRoots( - roots: Array, - ): Promise { - const watchmanRoots = new Map(); - await Promise.all( - roots.map(async root => { - const response = await cmd('watch-project', root); - const existing = watchmanRoots.get(response.watch); - // A root can only be filtered if it was never seen with a - // relative_path before. - const canBeFiltered = !existing || existing.length > 0; - - if (canBeFiltered) { - if (response.relative_path) { - watchmanRoots.set( - response.watch, - (existing || []).concat(response.relative_path), - ); - } else { - // Make the filter directories an empty array to signal that this - // root was already seen and needs to be watched for all files or - // directories. - watchmanRoots.set(response.watch, []); - } + // Clone the clockspec cache to avoid mutations during the watch phase. + const clocks = new Map(data.clocks); + + /** + * Fetch an array of files that match the given expression and are contained + * by the given `watchRoot` (with directory filters applied). + * + * When the `watchRoot` has a cached Watchman clockspec, only changed files + * are returned. The cloned clockspec cache is updated on every query. + * + * The given `watchRoot` must be absolute. + */ + const queryRequest = async ( + watchRoot: Config.Path, + dirs: Array, + expression: Array, + ) => { + if (dirs.length) { + expression = [ + 'allof', + ['anyof', ...dirs.map(dir => ['dirname', dir])], + expression, + ]; + } + + const relativeRoot = fastPath.relative(rootDir, watchRoot) || '.'; + const response = await cmd('query', watchRoot, { + expression, + fields, + // Use the cached clockspec since `queryRequest` is called twice per root. + since: data.clocks.get(relativeRoot), + }); + + if ('warning' in response) { + console.warn('watchman warning:', response.warning); + } + + clocks.set(relativeRoot, response.clock); + return response; + }; + + // The cached array of project roots. + const roots: Config.Path[] = []; + const isValidRoot = (root: Config.Path) => + !root.includes(NODE_MODULES) && + !roots.includes(root) && + !roots.find(root => root.startsWith(root + path.sep)); + + // Ensure every configured root is valid. + options.roots + .map(root => realpath(root)) + .sort((a, b) => a.length - b.length) + .forEach(root => { + if (isValidRoot(root)) { + roots.push(root); + } + }); + + // Ensure every linked root is searched if valid. + data.roots.forEach(root => { + root = path.resolve(rootDir, root); + if (isValidRoot(root)) { + try { + if (fs.statSync(root).isDirectory()) { + roots.push(root); } - }), + } catch (e) {} + } + }); + + // Track which directories share a "watch root" (a common ancestor identified + // by Watchman). Use an empty array to denote a project root that is its own + // watch root. These arrays are used as "directory filters" in the 2nd phase. + const watched = new Map(); + + // Store newfound dependency links in a local variable until the crawl phase, + // which is when we can know if Watchman is a fresh instance. + const dependencyLinks = new Map(); + + /** + * Look for linked dependencies in every "node_modules" directory within the + * given root. Repeat for every linked dependency found that resolves to a + * directory which exists outside of all known project roots. + */ + const findDependencyLinks = async (root: Config.Path) => { + const response = await cmd('watch-project', root); + const watchRoot = normalizePathSep(response.watch); + + let dirs = watched.get(watchRoot); + if (!dirs) { + watched.set(watchRoot, (dirs = [])); + } else if (!dirs.length) { + return; // Ensure no directory filters are used. + } + + const dir = normalizePathSep(response.relative_path || ''); + if (dir) { + if (dirs.includes(dir)) { + return; // Avoid crawling the same directory twice. + } + dirs.push(dir); + } else { + // Ensure no directory filters are used. + dirs.length = 0; + } + + // Search every "node_modules" directory in the entire tree. + // It should be faster for Watchman to do this in one fell swoop. + const queryResponse = await queryRequest( + watchRoot, + dir ? [dir] : [], + nodeModulesExpression, ); - return watchmanRoots; - } - async function queryWatchmanForDirs(rootProjectDirMappings: WatchmanRoots) { - const files = new Map(); - let isFresh = false; await Promise.all( - Array.from(rootProjectDirMappings).map( - async ([root, directoryFilters]) => { - const expression = Array.from(defaultWatchExpression); - const glob = []; - - if (directoryFilters.length > 0) { - expression.push([ - 'anyof', - ...directoryFilters.map(dir => ['dirname', dir]), - ]); - - for (const directory of directoryFilters) { - for (const extension of extensions) { - glob.push(`${directory}/**/*.${extension}`); - } - } - } else { - for (const extension of extensions) { - glob.push(`**/*.${extension}`); - } - } + queryResponse.files.map(async (link: WatchmanFile) => { + const name = normalizePathSep(link.name); + const linkPath = path.join(watchRoot, name); + const relativePath = fastPath.relative(rootDir, linkPath); + + if (!link.exists || ignore(linkPath)) { + data.links.delete(relativePath); + return; + } - const relativeRoot = fastPath.relative(rootDir, root); - const query = clocks.has(relativeRoot) - ? // Use the `since` generator if we have a clock available - {expression, fields, since: clocks.get(relativeRoot)} - : // Otherwise use the `glob` filter - {expression, fields, glob}; + let target; + try { + target = realpath(linkPath); + } catch (e) { + return; // Skip broken symlinks. + } - const response = await cmd('query', root, query); + const metaData = data.links.get(relativePath); + const mtime = testModified(metaData, link.mtime_ms); + dependencyLinks.set( + relativePath, + mtime !== 0 ? [fastPath.relative(rootDir, target), mtime] : metaData!, + ); - if ('warning' in response) { - console.warn('watchman warning: ', response.warning); - } - - isFresh = isFresh || response.is_fresh_instance; - files.set(root, response); - }, - ), + if (fs.statSync(target).isDirectory() && isValidRoot(target)) { + roots.push(target); + await findDependencyLinks(target); + } + }), ); + }; - return { - files, - isFresh, - }; - } - - let files = data.files; + let isFresh = false; let removedFiles = new Map(); const changedFiles = new Map(); - let watchmanFiles: Map; - let isFresh = false; try { - const watchmanRoots = await getWatchmanRoots(roots); - const watchmanFileResults = await queryWatchmanForDirs(watchmanRoots); + await Promise.all(roots.map(findDependencyLinks)); + + const crawlExpression = [ + 'anyof', + ['type', 'l'], + [ + 'allof', + ['type', 'f'], + ['anyof', ...extensions.map(extension => ['suffix', extension])], + ], + ]; + + const watchRoots = Array.from(watched.keys()); + const crawled = await Promise.all( + watchRoots.map(async watchRoot => { + const queryResponse = await queryRequest( + watchRoot, + watched.get(watchRoot)!, + crawlExpression, + ); + if (!isFresh) { + isFresh = queryResponse.is_fresh_instance; + } + return queryResponse.files; + }), + ); - // Reset the file map if watchman was restarted and sends us a list of - // files. - if (watchmanFileResults.isFresh) { - files = new Map(); + // Reset the file map if Watchman refreshed. + if (isFresh) { removedFiles = new Map(data.files); - isFresh = true; + data.files = new Map(); + data.links = dependencyLinks; + } else { + dependencyLinks.forEach((link, linkPath) => { + data.links.set(linkPath, link); + }); } - watchmanFiles = watchmanFileResults.files; - } finally { - client.end(); - } - - if (clientError) { - throw clientError; - } - - // TODO: remove non-null - for (const [watchRoot, response] of watchmanFiles!) { - const fsRoot = normalizePathSep(watchRoot); - const relativeFsRoot = fastPath.relative(rootDir, fsRoot); - clocks.set(relativeFsRoot, response.clock); - - for (const fileData of response.files) { - const filePath = fsRoot + path.sep + normalizePathSep(fileData.name); - const relativeFilePath = fastPath.relative(rootDir, filePath); - const existingFileData = data.files.get(relativeFilePath); - - // If watchman is fresh, the removed files map starts with all files - // and we remove them as we verify they still exist. - if (isFresh && existingFileData && fileData.exists) { - removedFiles.delete(relativeFilePath); - } + // Update the file map and symlink map. + crawled.forEach((files: WatchmanFile[], i) => { + const watchRoot = watchRoots[i]; + for (const file of files) { + const name = normalizePathSep(file.name); + const filePath = path.join(watchRoot, name); + const relativeFilePath = fastPath.relative(rootDir, filePath); + const isLinked = file.type === 'l'; + + const existingFiles: Map = isLinked + ? data.links + : data.files; + + const existingFile = existingFiles.get(relativeFilePath); + if (isFresh && existingFile && file.exists) { + // If watchman is fresh, the removed files map starts with all files + // and we remove them as we verify they still exist. + removedFiles.delete(relativeFilePath); + } - if (!fileData.exists) { - // No need to act on files that do not exist and were not tracked. - if (existingFileData) { - files.delete(relativeFilePath); + if (!file.exists || ignore(filePath)) { + // No need to act on files that do not exist and were not tracked. + if (existingFile) { + existingFiles.delete(relativeFilePath); - // If watchman is not fresh, we will know what specific files were - // deleted since we last ran and can track only those files. - if (!isFresh) { - removedFiles.set(relativeFilePath, existingFileData); + // If watchman is not fresh, we will know what specific files were + // deleted since we last ran and can track only those files. + if (!isFresh) { + removedFiles.set(relativeFilePath, existingFile); + } } + continue; } - } else if (!ignore(filePath)) { - const mtime = - typeof fileData.mtime_ms === 'number' - ? fileData.mtime_ms - : fileData.mtime_ms.toNumber(); - const size = fileData.size; - - let sha1hex = fileData['content.sha1hex']; - if (typeof sha1hex !== 'string' || sha1hex.length !== 40) { + + let sha1hex = file['content.sha1hex'] || null; + if (sha1hex && sha1hex.length !== 40) { sha1hex = null; } - let nextData: FileMetaData; - - if (existingFileData && existingFileData[H.MTIME] === mtime) { - nextData = existingFileData; - } else if ( - existingFileData && - sha1hex && - existingFileData[H.SHA1] === sha1hex - ) { - nextData = [ - existingFileData[0], - mtime, - existingFileData[2], - existingFileData[3], - existingFileData[4], - existingFileData[5], - ]; - } else { - // See ../constants.ts - nextData = ['', mtime, size, 0, '', sha1hex]; - } + const mtime = testModified(existingFile, file.mtime_ms); + if (mtime !== 0) { + if (isLinked) { + // See ../constants.ts + existingFiles.set(relativeFilePath, [undefined, mtime]); + continue; + } - const mappings = options.mapper ? options.mapper(filePath) : null; - - if (mappings) { - for (const absoluteVirtualFilePath of mappings) { - if (!ignore(absoluteVirtualFilePath)) { - const relativeVirtualFilePath = fastPath.relative( - rootDir, - absoluteVirtualFilePath, - ); - files.set(relativeVirtualFilePath, nextData); - changedFiles.set(relativeVirtualFilePath, nextData); + let nextData: FileMetaData; + if (sha1hex && existingFile && existingFile[H.SHA1] === sha1hex) { + nextData = [ + existingFile[0], + mtime, + existingFile[2], + existingFile[3], + existingFile[4], + existingFile[5], + ]; + } else { + // See ../constants.ts + nextData = ['', mtime, file.size, 0, '', sha1hex]; + } + + const mappings = options.mapper ? options.mapper(filePath) : null; + + if (mappings) { + for (const absoluteVirtualFilePath of mappings) { + if (!ignore(absoluteVirtualFilePath)) { + const relativeVirtualFilePath = fastPath.relative( + rootDir, + absoluteVirtualFilePath, + ); + data.files.set(relativeVirtualFilePath, nextData); + changedFiles.set(relativeVirtualFilePath, nextData); + } } + } else { + data.files.set(relativeFilePath, nextData); + changedFiles.set(relativeFilePath, nextData); } - } else { - files.set(relativeFilePath, nextData); - changedFiles.set(relativeFilePath, nextData); } } - } + }); + } finally { + client.end(); + } + if (clientError) { + throw clientError; } - data.files = files; + data.roots = roots.map(root => fastPath.relative(rootDir, root)); + data.clocks = clocks; return { changedFiles: isFresh ? undefined : changedFiles, hasteMap: data, removedFiles, }; }; + +/** + * Check if the file data has been modified since last cached. + * + * Returns the 2nd argument if modified, else zero. + */ +function testModified( + metaData: FileMetaData | LinkMetaData | undefined, + mtime: number | {toNumber(): number}, +) { + if (typeof mtime !== 'number') { + mtime = mtime.toNumber(); + } + return !metaData || metaData[H.MTIME] !== mtime ? mtime : 0; +} diff --git a/packages/jest-haste-map/src/index.ts b/packages/jest-haste-map/src/index.ts index 4a14bebc2eee..c8e634ea8e25 100644 --- a/packages/jest-haste-map/src/index.ts +++ b/packages/jest-haste-map/src/index.ts @@ -176,6 +176,8 @@ const getWhiteList = (list: Array | undefined): RegExp | null => { * type HasteMap = { * clocks: WatchmanClocks, * files: {[filepath: string]: FileMetaData}, + * links: {[filepath: string]: LinkMetaData}, + * roots: string[], * map: {[id: string]: ModuleMapItem}, * mocks: {[id: string]: string}, * } @@ -361,6 +363,7 @@ class HasteMap extends EventEmitter { const rootDir = this._options.rootDir; const hasteFS = new HasteFS({ files: hasteMap.files, + links: hasteMap.links, rootDir, }); const moduleMap = new HasteModuleMap({ @@ -817,8 +820,7 @@ class HasteMap extends EventEmitter { let mustCopy = true; const createWatcher = (root: Config.Path): Promise => { - // @ts-ignore: TODO how? "Cannot use 'new' with an expression whose type lacks a call or construct signature." - const watcher = new Watcher(root, { + const watcher = new Watcher(path.resolve(rootDir, root), { dot: false, glob: extensions.map(extension => '**/*.' + extension), ignored: ignorePattern, @@ -845,6 +847,7 @@ class HasteMap extends EventEmitter { eventsQueue, hasteFS: new HasteFS({ files: hasteMap.files, + links: hasteMap.links, rootDir, }), moduleMap: new HasteModuleMap({ @@ -897,6 +900,8 @@ class HasteMap extends EventEmitter { clocks: new Map(hasteMap.clocks), duplicates: new Map(hasteMap.duplicates), files: new Map(hasteMap.files), + links: new Map(hasteMap.links), + roots: [...hasteMap.roots], map: new Map(hasteMap.map), mocks: new Map(hasteMap.mocks), }; @@ -987,11 +992,9 @@ class HasteMap extends EventEmitter { }; this._changeInterval = setInterval(emitChange, CHANGE_INTERVAL); - return Promise.all(this._options.roots.map(createWatcher)).then( - watchers => { - this._watchers = watchers; - }, - ); + return Promise.all(hasteMap.roots.map(createWatcher)).then(watchers => { + this._watchers = watchers; + }); } /** @@ -1109,6 +1112,8 @@ class HasteMap extends EventEmitter { clocks: new Map(), duplicates: new Map(), files: new Map(), + links: new Map(), + roots: [], map: new Map(), mocks: new Map(), }; diff --git a/packages/jest-haste-map/src/types.ts b/packages/jest-haste-map/src/types.ts index 9f776911a225..a6edeab31ede 100644 --- a/packages/jest-haste-map/src/types.ts +++ b/packages/jest-haste-map/src/types.ts @@ -45,6 +45,7 @@ export type HasteImpl = { }; export type FileData = Map; +export type LinkData = Map; export type FileMetaData = [ /* id */ string, @@ -55,6 +56,11 @@ export type FileMetaData = [ /* sha1 */ string | null | undefined, ]; +export type LinkMetaData = [ + /* target */ string | undefined, + /* mtime */ number +]; + export type MockData = Map; export type ModuleMapData = Map; export type WatchmanClocks = Map; @@ -67,6 +73,8 @@ export type InternalHasteMap = { clocks: WatchmanClocks; duplicates: DuplicatesIndex; files: FileData; + links: LinkData; + roots: Config.Path[]; map: ModuleMapData; mocks: MockData; };