From 5bf3b27c4ba50d67ba34bdf3e6edcaaf0d587368 Mon Sep 17 00:00:00 2001 From: Ahn Date: Thu, 3 Dec 2020 11:34:44 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20burst=20cache=20when=20file=E2=80=99s=20?= =?UTF-8?q?deps=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Construct cache key with additional information is resolved module name + resolved module's last modified time. When one of imported modules of a file changes, the file needs to be reprocessed to have type check properly Closes #2118 Closes #1122 Closes #943 --- .../__snapshots__/logger.test.ts.snap | 18 +- .../__snapshots__/ts-compiler.spec.ts.snap | 62 ------ src/compiler/ts-compiler.spec.ts | 129 +----------- src/compiler/ts-compiler.ts | 128 +++--------- src/compiler/ts-jest-compiler.ts | 6 +- src/config/config-set.ts | 31 ++- src/ts-jest-transformer.spec.ts | 196 ++++++++++++++++-- src/ts-jest-transformer.ts | 142 ++++++++++--- src/types.ts | 3 + 9 files changed, 351 insertions(+), 364 deletions(-) diff --git a/e2e/__tests__/__snapshots__/logger.test.ts.snap b/e2e/__tests__/__snapshots__/logger.test.ts.snap index 8b0437605b..07f129be7a 100644 --- a/e2e/__tests__/__snapshots__/logger.test.ts.snap +++ b/e2e/__tests__/__snapshots__/logger.test.ts.snap @@ -20,14 +20,18 @@ Array [ "[level:20] normalized typescript config via ts-jest option", "[level:20] normalized custom AST transformers via ts-jest option", "[level:20] file caching disabled", + "[level:20] created language service", "[level:20] computing cache key for /Hello.spec.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.spec.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.spec.ts", - "[level:20] created language service", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] visitSourceFileNode(): hoisting", "[level:20] getCompiledOutput(): computing diagnostics using language service", "[level:20] computing cache key for /Hello.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.ts", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", @@ -60,15 +64,19 @@ Array [ "[level:20] normalized typescript config via ts-jest option", "[level:20] normalized custom AST transformers via ts-jest option", "[level:20] file caching disabled", + "[level:20] created language service", "[level:20] computing cache key for /Hello.spec.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.spec.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.spec.ts", - "[level:20] created language service", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] visitSourceFileNode(): hoisting", "[level:20] getCompiledOutput(): computing diagnostics using language service", "[level:20] calling babel-jest processor", "[level:20] computing cache key for /Hello.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.ts", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", @@ -103,15 +111,19 @@ Array [ "[level:20] normalized typescript config via ts-jest option", "[level:20] normalized custom AST transformers via ts-jest option", "[level:20] file caching disabled", + "[level:20] created language service", "[level:20] computing cache key for /Hello.spec.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.spec.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.spec.ts", - "[level:20] created language service", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] visitSourceFileNode(): hoisting", "[level:20] getCompiledOutput(): computing diagnostics using language service", "[level:20] calling babel-jest processor", "[level:20] computing cache key for /Hello.ts", + "[level:20] getting resolved modules from TypeScript API for /Hello.ts", + "[level:20] updateMemoryCache: update memory cache for language service", "[level:20] processing /Hello.ts", "[level:20] getCompiledOutput(): compiling using language service", "[level:20] updateMemoryCache: update memory cache for language service", diff --git a/src/compiler/__snapshots__/ts-compiler.spec.ts.snap b/src/compiler/__snapshots__/ts-compiler.spec.ts.snap index f10e600b25..4b61a77801 100644 --- a/src/compiler/__snapshots__/ts-compiler.spec.ts.snap +++ b/src/compiler/__snapshots__/ts-compiler.spec.ts.snap @@ -36,49 +36,6 @@ exports[`TsCompiler isolatedModule false allowJs option should compile js file f ================================================================================ `; -exports[`TsCompiler isolatedModule false diagnostics should not report diagnostics for test file which doesn't exist when compiling import module file 1`] = ` -Array [ - "[level:20] getCompiledOutput(): compiling using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] visitSourceFileNode(): hoisting -", - "[level:20] getCompiledOutput(): computing diagnostics using language service -", -] -`; - -exports[`TsCompiler isolatedModule false diagnostics should only report diagnostics for imported modules but not test files without cache 1`] = ` -Array [ - "[level:20] getCompiledOutput(): compiling using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] visitSourceFileNode(): hoisting -", - "[level:20] getCompiledOutput(): computing diagnostics using language service -", -] -`; - -exports[`TsCompiler isolatedModule false diagnostics should report diagnostics for imported modules as well as test files which use imported modules with cache 1`] = ` -Array [ - "[level:20] getCompiledOutput(): compiling using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] visitSourceFileNode(): hoisting -", - "[level:20] getCompiledOutput(): computing diagnostics using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] getCompiledOutput(): computing diagnostics using language service for test file which uses the module -", -] -`; - exports[`TsCompiler isolatedModule false diagnostics should throw error when cannot compile 1`] = ` "Unable to require \`.d.ts\` file for file: test-cannot-compile.d.ts. This is usually the result of a faulty configuration or import. Make sure there is a \`.js\`, \`.json\` or another executable extension available alongside \`test-cannot-compile.d.ts\`." @@ -97,25 +54,6 @@ Array [ ] `; -exports[`TsCompiler isolatedModule false diagnostics shouldn't report diagnostics for test file name that has been type checked before 1`] = ` -Array [ - "[level:20] getCompiledOutput(): compiling using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] visitSourceFileNode(): hoisting -", - "[level:20] getCompiledOutput(): computing diagnostics using language service -", - "[level:20] getCompiledOutput(): compiling using language service -", - "[level:20] updateMemoryCache: update memory cache for language service -", - "[level:20] visitSourceFileNode(): hoisting -", -] -`; - exports[`TsCompiler isolatedModule false jsx option should compile tsx file for jsx preserve 1`] = ` ===[ FILE: test-jsx.tsx ]======================================================= "use strict"; diff --git a/src/compiler/ts-compiler.spec.ts b/src/compiler/ts-compiler.spec.ts index 1ffcf0a14c..f11107f712 100644 --- a/src/compiler/ts-compiler.spec.ts +++ b/src/compiler/ts-compiler.spec.ts @@ -1,7 +1,5 @@ -import { readFileSync, renameSync } from 'fs' +import { readFileSync } from 'fs' import { LogLevels } from 'bs-logger' -import { removeSync } from 'fs-extra' -import { join } from 'path' import { TS_JEST_OUT_DIR } from '../config/config-set' import { makeCompiler } from '../__helpers__/fakers' @@ -284,131 +282,6 @@ const t: string = f(5) const importedFileName = require.resolve('../__mocks__/thing.ts') const importedFileContent = readFileSync(importedFileName, 'utf-8') - it(`should report diagnostics for imported modules as well as test files which use imported modules with cache`, async () => { - const testFileName = require.resolve('../__mocks__/thing1.spec.ts') - const testFileContent = readFileSync(testFileName, 'utf-8') - const cacheDir = join(process.cwd(), 'tmp') - /** - * Run the 1st compilation with Promise resolve setTimeout to stimulate 2 different test runs to test cached - * resolved modules - */ - async function firstCompile() { - return new Promise((resolve) => { - const compiler1 = makeCompiler({ - jestConfig: { - cache: true, - cacheDirectory: cacheDir, - }, - tsJestConfig: baseTsJestConfig, - }) - - logTarget.clear() - compiler1.getCompiledOutput(testFileContent, testFileName) - - // probably 300ms is enough to stimulate 2 separated runs after each other - setTimeout(() => resolve(), 300) - }) - } - - await firstCompile() - - const compiler2 = makeCompiler({ - jestConfig: { - cache: true, - cacheDirectory: cacheDir, - }, - tsJestConfig: baseTsJestConfig, - }) - logTarget.clear() - - compiler2.getCompiledOutput(importedFileContent, importedFileName) - - expect(logTarget.filteredLines(LogLevels.debug, Infinity)).toMatchSnapshot() - - removeSync(cacheDir) - }) - - it(`should not report diagnostics for test file which doesn't exist when compiling import module file`, async () => { - const testFileName = require.resolve('../__mocks__/thing.spec.ts') - const testFileContent = readFileSync(testFileName, 'utf-8') - const cacheDir = join(process.cwd(), 'tmp') - /** - * Run the 1st compilation with Promise resolve setTimeout to stimulate 2 different test runs to test cached - * resolved modules - */ - async function firstCompile() { - return new Promise((resolve) => { - const compiler1 = makeCompiler({ - jestConfig: { - cache: true, - cacheDirectory: cacheDir, - }, - tsJestConfig: baseTsJestConfig, - }) - - logTarget.clear() - compiler1.getCompiledOutput(testFileContent, testFileName) - - // probably 300ms is enough to stimulate 2 separated runs after each other - setTimeout(() => resolve(), 300) - }) - } - - await firstCompile() - - const newTestFileName = testFileName.replace('thing', 'thing2') - renameSync(testFileName, newTestFileName) - - const compiler2 = makeCompiler({ - jestConfig: { - cache: true, - cacheDirectory: cacheDir, - }, - tsJestConfig: baseTsJestConfig, - }) - logTarget.clear() - - compiler2.getCompiledOutput(importedFileContent, importedFileName) - - expect(logTarget.filteredLines(LogLevels.debug, Infinity)).toMatchSnapshot() - - renameSync(newTestFileName, testFileName) - removeSync(cacheDir) - }) - - it(`should only report diagnostics for imported modules but not test files without cache`, () => { - const testFileName = require.resolve('../__mocks__/thing1.spec.ts') - const testFileContent = readFileSync(testFileName, 'utf-8') - const compiler1 = makeCompiler({ - tsJestConfig: baseTsJestConfig, - }) - logTarget.clear() - compiler1.getCompiledOutput(testFileContent, testFileName) - - const compiler2 = makeCompiler({ - tsJestConfig: baseTsJestConfig, - }) - logTarget.clear() - - compiler2.getCompiledOutput(importedFileContent, importedFileName) - - expect(logTarget.filteredLines(LogLevels.debug, Infinity)).toMatchSnapshot() - }) - - it(`shouldn't report diagnostics for test file name that has been type checked before`, () => { - const testFileName = require.resolve('../__mocks__/thing1.spec.ts') - const testFileContent = readFileSync(testFileName, 'utf-8') - const compiler1 = makeCompiler({ - tsJestConfig: baseTsJestConfig, - }) - logTarget.clear() - - compiler1.getCompiledOutput(testFileContent, testFileName) - compiler1.getCompiledOutput(testFileContent, testFileName) - - expect(logTarget.filteredLines(LogLevels.debug, Infinity)).toMatchSnapshot() - }) - it(`shouldn't report diagnostics when file name doesn't match diagnostic file pattern`, () => { const compiler = makeCompiler({ tsJestConfig: { diff --git a/src/compiler/ts-compiler.ts b/src/compiler/ts-compiler.ts index 8d9c66ffc8..0722fc4c1c 100644 --- a/src/compiler/ts-compiler.ts +++ b/src/compiler/ts-compiler.ts @@ -1,8 +1,6 @@ -import { LogContexts, LogLevels, Logger } from 'bs-logger' -import { basename, join, normalize, relative } from 'path' -import { existsSync, readFileSync, writeFile } from 'fs' +import { LogContexts, Logger, LogLevels } from 'bs-logger' import memoize from 'lodash.memoize' -import mkdirp from 'mkdirp' +import { basename, normalize, relative } from 'path' import type { EmitOutput, LanguageService, @@ -15,25 +13,15 @@ import type { import { updateOutput } from './compiler-utils' import type { ConfigSet } from '../config/config-set' import { LINE_FEED } from '../constants' -import type { CompilerInstance, TTypeScript } from '../types' +import type { CompilerInstance, ResolvedModulesMap, TTypeScript } from '../types' import { rootLogger } from '../utils/logger' -import { parse, stringify } from '../utils/json' import { Errors, interpolate } from '../utils/messages' -import { sha1 } from '../utils/sha1' - -/** where key is filepath */ -type TSFiles = Map interface TSFile { text?: string version: number } -interface MemoryCache { - resolvedModules: Map - files: TSFiles -} - /** * @internal */ @@ -41,14 +29,9 @@ export class TsCompiler implements CompilerInstance { private readonly _logger: Logger private readonly _ts: TTypeScript private readonly _parsedTsConfig: ParsedCommandLine - private readonly _memoryHost: MemoryCache = { - files: new Map(), - resolvedModules: new Map(), - } - private readonly _diagnosedFiles: string[] = [] + private readonly _cacheFS: Map = new Map() private _cachedReadFile: any private _projectVersion = 1 - private _tsResolvedModulesCachePath: string | undefined private _languageService: LanguageService | undefined constructor(private readonly configSet: ConfigSet) { @@ -61,27 +44,16 @@ export class TsCompiler implements CompilerInstance { } private _createLanguageService(): void { - const cacheDir = this.configSet.tsCacheDir const serviceHostTraceCtx = { namespace: 'ts:serviceHost', call: null, [LogContexts.logLevel]: LogLevels.trace, } - if (cacheDir) { - // Make sure the cache directory exists before continuing. - mkdirp.sync(cacheDir) - this._tsResolvedModulesCachePath = join(cacheDir, sha1('ts-jest-resolved-modules', '\x00')) - try { - /* istanbul ignore next (already covered with unit test) */ - const cachedTSResolvedModules = readFileSync(this._tsResolvedModulesCachePath, 'utf-8') - this._memoryHost.resolvedModules = new Map(parse(cachedTSResolvedModules)) - } catch (e) {} - } // Initialize memory cache for typescript compiler this._parsedTsConfig.fileNames .filter((fileName) => !this.configSet.isTestFile(fileName)) .forEach((fileName) => { - this._memoryHost.files.set(fileName, { + this._cacheFS.set(fileName, { version: 0, }) }) @@ -103,10 +75,10 @@ export class TsCompiler implements CompilerInstance { /* istanbul ignore next */ const serviceHost: LanguageServiceHost = { getProjectVersion: () => String(this._projectVersion), - getScriptFileNames: () => [...this._memoryHost.files.keys()], + getScriptFileNames: () => [...this._cacheFS.keys()], getScriptVersion: (fileName: string) => { const normalizedFileName = normalize(fileName) - const version = this._memoryHost.files.get(normalizedFileName)?.version + const version = this._cacheFS.get(normalizedFileName)?.version // We need to return `undefined` and not a string here because TypeScript will use // `getScriptVersion` and compare against their own version - which can be `undefined`. @@ -123,12 +95,12 @@ export class TsCompiler implements CompilerInstance { // Read contents from TypeScript memory cache. if (!hit) { - this._memoryHost.files.set(normalizedFileName, { + this._cacheFS.set(normalizedFileName, { text: this._cachedReadFile(normalizedFileName), version: 1, }) } - const contents = this._memoryHost.files.get(normalizedFileName)?.text + const contents = this._cacheFS.get(normalizedFileName)?.text if (contents === undefined) return @@ -145,30 +117,18 @@ export class TsCompiler implements CompilerInstance { getCompilationSettings: () => this._parsedTsConfig.options, getDefaultLibFileName: () => this._ts.getDefaultLibFilePath(this._parsedTsConfig.options), getCustomTransformers: () => this.configSet.customTransformers, - resolveModuleNames: (moduleNames: string[], containingFile: string): (ResolvedModuleFull | undefined)[] => { - const normalizedContainingFile = normalize(containingFile) - const currentResolvedModules = this._memoryHost.resolvedModules.get(normalizedContainingFile) ?? [] - - return moduleNames.map((moduleName) => { - const resolveModuleName = this._ts.resolveModuleName( + resolveModuleNames: (moduleNames: string[], containingFile: string): (ResolvedModuleFull | undefined)[] => + moduleNames.map((moduleName) => { + const { resolvedModule } = this._ts.resolveModuleName( moduleName, containingFile, this._parsedTsConfig.options, moduleResolutionHost, moduleResolutionCache, ) - const resolvedModule = resolveModuleName.resolvedModule - if (this.configSet.isTestFile(normalizedContainingFile) && resolvedModule) { - const normalizedResolvedFileName = normalize(resolvedModule.resolvedFileName) - if (!currentResolvedModules.includes(normalizedResolvedFileName)) { - currentResolvedModules.push(normalizedResolvedFileName) - this._memoryHost.resolvedModules.set(normalizedContainingFile, currentResolvedModules) - } - } return resolvedModule - }) - }, + }), } this._logger.debug('created language service') @@ -176,6 +136,13 @@ export class TsCompiler implements CompilerInstance { this._languageService = this._ts.createLanguageService(serviceHost, this._ts.createDocumentRegistry()) } + getResolvedModulesMap(fileContent: string, fileName: string): ResolvedModulesMap { + this._updateMemoryCache(fileContent, fileName) + + // See https://github.com/microsoft/TypeScript/blob/master/src/compiler/utilities.ts#L164 + return (this._languageService?.getProgram()?.getSourceFile(fileName) as any).resolvedModules + } + getCompiledOutput(fileContent: string, fileName: string): string { if (this._languageService) { this._logger.debug({ fileName }, 'getCompiledOutput(): compiling using language service') @@ -183,52 +150,10 @@ export class TsCompiler implements CompilerInstance { // Must set memory cache before attempting to compile this._updateMemoryCache(fileContent, fileName) const output: EmitOutput = this._languageService.getEmitOutput(fileName) - /* istanbul ignore next */ - if (this._tsResolvedModulesCachePath) { - // Cache resolved modules to disk so next run can reuse it - void (async () => { - // eslint-disable-next-line @typescript-eslint/await-thenable - await writeFile( - this._tsResolvedModulesCachePath as string, - stringify([...this._memoryHost.resolvedModules]), - () => {}, - ) - })() - } - /** - * There might be a chance that test files are type checked even before jest executes them, we don't need to do - * type check again - */ - if (!this._diagnosedFiles.includes(fileName)) { - this._logger.debug({ fileName }, 'getCompiledOutput(): computing diagnostics using language service') - this._doTypeChecking(fileName) - } - /* istanbul ignore next (already covered with unit tests) */ - if (!this.configSet.isTestFile(fileName)) { - for (const [testFileName, resolvedModules] of this._memoryHost.resolvedModules.entries()) { - // Only do type checking for test files which haven't been type checked before as well as the file must exist - if ( - resolvedModules.includes(fileName) && - !this._diagnosedFiles.includes(testFileName) && - existsSync(testFileName) - ) { - const testFileContent = this._memoryHost.files.get(testFileName)?.text - if (!testFileContent) { - // Must set memory cache before attempting to get diagnostics - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this._updateMemoryCache(this._cachedReadFile(testFileName)!, testFileName) - } - - this._logger.debug( - { testFileName }, - 'getCompiledOutput(): computing diagnostics using language service for test file which uses the module', - ) + this._logger.debug({ fileName }, 'getCompiledOutput(): computing diagnostics using language service') - this._doTypeChecking(testFileName) - } - } - } + this._doTypeChecking(fileName) /* istanbul ignore next (this should never happen but is kept for security) */ if (output.emitSkipped) { throw new TypeError(`${relative(this.configSet.cwd, fileName)}: Emit skipped for language service`) @@ -263,7 +188,7 @@ export class TsCompiler implements CompilerInstance { private _isFileInCache(fileName: string): boolean { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return this._memoryHost.files.has(fileName) && this._memoryHost.files.get(fileName)!.version !== 0 + return this._cacheFS.has(fileName) && this._cacheFS.get(fileName)!.version !== 0 } /* istanbul ignore next */ @@ -273,18 +198,18 @@ export class TsCompiler implements CompilerInstance { let shouldIncrementProjectVersion = false const hit = this._isFileInCache(fileName) if (!hit) { - this._memoryHost.files.set(fileName, { + this._cacheFS.set(fileName, { text: contents, version: 1, }) shouldIncrementProjectVersion = true } else { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const cachedFileName = this._memoryHost.files.get(fileName)! + const cachedFileName = this._cacheFS.get(fileName)! const previousContents = cachedFileName.text // Avoid incrementing cache when nothing has changed. if (previousContents !== contents) { - this._memoryHost.files.set(fileName, { + this._cacheFS.set(fileName, { text: contents, version: cachedFileName.version + 1, }) @@ -311,7 +236,6 @@ export class TsCompiler implements CompilerInstance { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion this._languageService!.getSyntacticDiagnostics(fileName), ) - this._diagnosedFiles.push(fileName) // will raise or just warn diagnostics depending on config this.configSet.raiseDiagnostics(diagnostics, fileName, this._logger) } diff --git a/src/compiler/ts-jest-compiler.ts b/src/compiler/ts-jest-compiler.ts index d498251f38..667ce339e7 100644 --- a/src/compiler/ts-jest-compiler.ts +++ b/src/compiler/ts-jest-compiler.ts @@ -1,6 +1,6 @@ import { TsCompiler } from './ts-compiler' import type { ConfigSet } from '../config/config-set' -import type { CompilerInstance } from '../types' +import type { CompilerInstance, ResolvedModulesMap } from '../types' /** * @internal @@ -13,6 +13,10 @@ export class TsJestCompiler implements CompilerInstance { this._compilerInstance = new TsCompiler(this.configSet) } + getResolvedModulesMap(fileContent: string, fileName: string): ResolvedModulesMap { + return this._compilerInstance.getResolvedModulesMap(fileContent, fileName) + } + getCompiledOutput(fileContent: string, fileName: string): string { return this._compilerInstance.getCompiledOutput(fileContent, fileName) } diff --git a/src/config/config-set.ts b/src/config/config-set.ts index 1943d68cd0..39fb736cbd 100644 --- a/src/config/config-set.ts +++ b/src/config/config-set.ts @@ -341,24 +341,23 @@ export class ConfigSet { private _resolveTsCacheDir(): void { if (!this._jestCfg.cache) { this.logger.debug('file caching disabled') + } else { + const cacheSuffix = sha1( + stringify({ + version: this.compilerModule.version, + digest: this.tsJestDigest, + compilerModule: this.compilerModule, + compilerOptions: this.parsedTsConfig.options, + isolatedModules: this.isolatedModules, + diagnostics: this._diagnostics, + }), + ) + const res = join(this._jestCfg.cacheDirectory, 'ts-jest', cacheSuffix.substr(0, 2), cacheSuffix.substr(2)) - return undefined - } - const cacheSuffix = sha1( - stringify({ - version: this.compilerModule.version, - digest: this.tsJestDigest, - compilerModule: this.compilerModule, - compilerOptions: this.parsedTsConfig.options, - isolatedModules: this.isolatedModules, - diagnostics: this._diagnostics, - }), - ) - const res = join(this._jestCfg.cacheDirectory, 'ts-jest', cacheSuffix.substr(0, 2), cacheSuffix.substr(2)) - - this.logger.debug({ cacheDirectory: res }, 'will use file caching') + this.logger.debug({ cacheDirectory: res }, 'will use file caching') - this.tsCacheDir = res + this.tsCacheDir = res + } } /** diff --git a/src/ts-jest-transformer.spec.ts b/src/ts-jest-transformer.spec.ts index 469a339e5c..68ec21e6ce 100644 --- a/src/ts-jest-transformer.spec.ts +++ b/src/ts-jest-transformer.spec.ts @@ -1,48 +1,140 @@ import { LogLevels } from 'bs-logger' -import { sep } from 'path' +import fs from 'fs' +import { removeSync, writeFileSync } from 'fs-extra' +import mkdirp from 'mkdirp' +import { join, sep } from 'path' +import { Extension, ResolvedModuleFull } from 'typescript' -import { logTargetMock } from './__helpers__/mocks' -import { TsJestTransformer } from './ts-jest-transformer' import { SOURCE_MAPPING_PREFIX } from './compiler/compiler-utils' import { TsJestCompiler } from './compiler/ts-jest-compiler' +import { ConfigSet } from './config/config-set' +import { logTargetMock } from './__helpers__/mocks' +import { CACHE_KEY_EL_SEPARATOR, TsJestTransformer } from './ts-jest-transformer' +import type { ResolvedModulesMap } from './types' +import { stringify } from './utils/json' +import { sha1 } from './utils/sha1' const logTarget = logTargetMock() +const cacheDir = join(process.cwd(), 'tmp') +const resolvedModule = { + resolvedFileName: join(__dirname, '__mocks__', 'thing.ts'), + extension: Extension.Ts, + isExternalLibraryImport: false, + packageId: undefined, +} beforeEach(() => { logTarget.clear() }) describe('TsJestTransformer', () => { - describe('configFor', () => { - test('should return the same config-set for same values with jest config string is not in configSetsIndex', () => { + describe('_configsFor', () => { + test('should return the same config set for same values with jest config string is not in configSetsIndex', () => { const obj1 = { cwd: '/foo/.', rootDir: '/bar//dummy/..', globals: {} } - const cs3 = new TsJestTransformer().configsFor(obj1 as any) + // @ts-expect-error testing purpose + const cs3 = new TsJestTransformer()._configsFor(obj1 as any) expect(cs3.cwd).toBe(`${sep}foo`) expect(cs3.rootDir).toBe(`${sep}bar`) }) - test('should return the same config-set for same values with jest config string in configSetsIndex', () => { + test('should return the same config set for same values with jest config string in configSetsIndex', () => { const obj1 = { cwd: '/foo/.', rootDir: '/bar//dummy/..', globals: {} } const obj2 = { ...obj1 } - const cs1 = new TsJestTransformer().configsFor(obj1 as any) - const cs2 = new TsJestTransformer().configsFor(obj2 as any) + // @ts-expect-error testing purpose + const cs1 = new TsJestTransformer()._configsFor(obj1 as any) + // @ts-expect-error testing purpose + const cs2 = new TsJestTransformer()._configsFor(obj2 as any) expect(cs1.cwd).toBe(`${sep}foo`) expect(cs1.rootDir).toBe(`${sep}bar`) expect(cs2).toBe(cs1) }) + + test(`should not read disk cache with isolatedModules true`, () => { + const tr = new TsJestTransformer() + const cs = new ConfigSet({ + testMatch: [], + testRegex: [], + globals: { 'ts-jest': { isolatedModules: true } }, + } as any) + const readFileSyncSpy = jest.spyOn(fs, 'readFileSync') + + // @ts-expect-error testing purpose + tr._getFsCachedResolvedModules(cs) + + expect(readFileSyncSpy).not.toHaveBeenCalled() + + readFileSyncSpy.mockRestore() + }) + + test(`should not read disk cache with isolatedModules false and no jest cache`, () => { + const tr = new TsJestTransformer() + const cs = new ConfigSet({ + testMatch: [], + testRegex: [], + globals: { 'ts-jest': { isolatedModules: false } }, + } as any) + const readFileSyncSpy = jest.spyOn(fs, 'readFileSync') + + // @ts-expect-error testing purpose + tr._getFsCachedResolvedModules(cs) + + expect(readFileSyncSpy).not.toHaveBeenCalled() + + readFileSyncSpy.mockRestore() + }) + + test(`should read disk cache with isolatedModules false and use jest cache`, () => { + const readFileSyncSpy = jest.spyOn(fs, 'readFileSync') + const fileName = 'foo.ts' + const tr = new TsJestTransformer() + const cs = new ConfigSet({ + cache: true, + cacheDirectory: cacheDir, + globals: { 'ts-jest': { isolatedModules: false } }, + testMatch: [], + testRegex: [], + } as any) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const tsCacheDir = cs.tsCacheDir! + const depGraphs: ResolvedModulesMap = new Map() + depGraphs.set(fileName, resolvedModule) + const resolvedModulesCacheDir = join(tsCacheDir, sha1('ts-jest-resolved-modules', CACHE_KEY_EL_SEPARATOR)) + mkdirp.sync(tsCacheDir) + writeFileSync(resolvedModulesCacheDir, stringify([...depGraphs])) + + // @ts-expect-error testing purpose + tr._getFsCachedResolvedModules(cs) + + // @ts-expect-error testing purpose + expect(tr._depGraphs.has(fileName)).toBe(true) + expect(readFileSyncSpy.mock.calls).toEqual(expect.arrayContaining([[resolvedModulesCacheDir, 'utf-8']])) + + removeSync(cacheDir) + }) }) describe('getCacheKey', () => { + let tr: TsJestTransformer + const input = { + fileContent: 'export default "foo"', + fileName: 'foo.ts', + jestConfigStr: '{"foo": "bar"}', + options: { config: { foo: 'bar', testMatch: [], testRegex: [] } as any, instrument: false, rootDir: '/foo' }, + } + const depGraphs: ResolvedModulesMap = new Map() + + beforeEach(() => { + depGraphs.clear() + tr = new TsJestTransformer() + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + test('should be different for each argument value', () => { - const tr = new TsJestTransformer() - const input = { - fileContent: 'export default "foo"', - fileName: 'foo.ts', - jestConfigStr: '{"foo": "bar"}', - options: { config: { foo: 'bar', testMatch: [], testRegex: [] } as any, instrument: false, rootDir: '/foo' }, - } const keys = [ tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options), tr.getCacheKey(input.fileContent, 'bar.ts', input.jestConfigStr, input.options), @@ -57,18 +149,86 @@ describe('TsJestTransformer', () => { // unique array should have same length expect(keys.filter((k, i, all) => all.indexOf(k) === i)).toHaveLength(keys.length) }) + + test('should be the same with the same file content', () => { + depGraphs.set(input.fileName, resolvedModule) + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + + const cacheKey1 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + const cacheKey2 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + + expect(cacheKey1).toEqual(cacheKey2) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledTimes(1) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledWith(input.fileContent, input.fileName) + }) + + test('should be different between isolatedModules true and isolatedModules false', () => { + depGraphs.set(input.fileName, resolvedModule) + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + + const cacheKey1 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, { + ...input.options, + config: { + ...input.options.config, + globals: { 'ts-jest': { isolatedModules: true } }, + }, + }) + + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + const tr1 = new TsJestTransformer() + const cacheKey2 = tr1.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledTimes(1) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledWith(input.fileContent, input.fileName) + expect(cacheKey1).not.toEqual(cacheKey2) + }) + + test('should be different with different file content for the same file', () => { + depGraphs.set(input.fileName, resolvedModule) + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + + const cacheKey1 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + const newFileContent = 'const foo = 1' + const cacheKey2 = tr.getCacheKey(newFileContent, input.fileName, input.jestConfigStr, input.options) + + expect(cacheKey1).not.toEqual(cacheKey2) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledTimes(2) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenNthCalledWith( + 1, + input.fileContent, + input.fileName, + ) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenNthCalledWith(2, newFileContent, input.fileName) + }) + + test('should be different with non existed imported modules', () => { + depGraphs.set(input.fileName, resolvedModule) + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(depGraphs) + + const cacheKey1 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + + jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false) + const cacheKey2 = tr.getCacheKey(input.fileContent, input.fileName, input.jestConfigStr, input.options) + + expect(cacheKey1).not.toEqual(cacheKey2) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledTimes(1) + expect(TsJestCompiler.prototype.getResolvedModulesMap).toHaveBeenCalledWith(input.fileContent, input.fileName) + }) }) describe('process', () => { - let tr!: any + let tr!: TsJestTransformer beforeEach(() => { tr = new TsJestTransformer() + jest.spyOn(TsJestCompiler.prototype, 'getResolvedModulesMap').mockReturnValueOnce(new Map()) }) test('should process input as stringified content with content matching stringifyContentPathRegex option', () => { - const fileContent = '

Hello World

' const filePath = 'foo.html' + const fileContent = '

Hello World

' const jestCfg = { globals: { 'ts-jest': { diff --git a/src/ts-jest-transformer.ts b/src/ts-jest-transformer.ts index ed90cb6e79..1bf182b821 100644 --- a/src/ts-jest-transformer.ts +++ b/src/ts-jest-transformer.ts @@ -1,23 +1,34 @@ import type { CacheKeyOptions, TransformedSource, Transformer, TransformOptions } from '@jest/transform' import type { Config } from '@jest/types' import type { Logger } from 'bs-logger' +import { existsSync, readFileSync, statSync, writeFile } from 'fs' +import mkdirp from 'mkdirp' +import { join } from 'path' +import { TsJestCompiler } from './compiler/ts-jest-compiler' import { ConfigSet } from './config/config-set' import { DECLARATION_TYPE_EXT, JS_JSX_REGEX, TS_TSX_REGEX } from './constants' -import { stringify } from './utils/json' +import { parse, stringify } from './utils/json' import { JsonableValue } from './utils/jsonable-value' import { rootLogger } from './utils/logger' import { Errors, interpolate } from './utils/messages' -import { TsJestCompiler } from './compiler/ts-jest-compiler' -import { sha1 } from './utils/sha1' import type { CompilerInstance } from './types' +import { sha1 } from './utils/sha1' interface CachedConfigSet { configSet: ConfigSet jestConfig: JsonableValue transformerCfgStr: string + compiler: CompilerInstance +} + +export interface DepGraphInfo { + fileContent: string + resolveModuleNames: string[] } +export const CACHE_KEY_EL_SEPARATOR = '\x00' + export class TsJestTransformer implements Transformer { /** * cache ConfigSet between test runs @@ -28,26 +39,26 @@ export class TsJestTransformer implements Transformer { /** * @internal */ - private static _compiler: CompilerInstance | undefined - protected readonly logger: Logger + private _compiler!: CompilerInstance + protected readonly _logger: Logger + protected _tsResolvedModulesCachePath: string | undefined protected _transformCfgStr!: string + protected _depGraphs: Map = new Map() constructor() { - this.logger = rootLogger.child({ namespace: 'ts-jest-transformer' }) + this._logger = rootLogger.child({ namespace: 'ts-jest-transformer' }) - this.logger.debug('created new transformer') + this._logger.debug('created new transformer') } - /** - * @public - */ - configsFor(jestConfig: Config.ProjectConfig): ConfigSet { + protected _configsFor(jestConfig: Config.ProjectConfig): ConfigSet { const ccs: CachedConfigSet | undefined = TsJestTransformer._cachedConfigSets.find( (cs) => cs.jestConfig.value === jestConfig, ) let configSet: ConfigSet if (ccs) { this._transformCfgStr = ccs.transformerCfgStr + this._compiler = ccs.compiler configSet = ccs.configSet } else { // try to look-it up by stringified version @@ -61,20 +72,18 @@ export class TsJestTransformer implements Transformer { // the config, and then it calls the transformer with the proper object serializedCcs.jestConfig.value = jestConfig this._transformCfgStr = serializedCcs.transformerCfgStr + this._compiler = serializedCcs.compiler configSet = serializedCcs.configSet } else { // create the new record in the index - this.logger.info('no matching config-set found, creating a new one') + this._logger.info('no matching config-set found, creating a new one') configSet = new ConfigSet(jestConfig) const jest = { ...jestConfig } - const globals = (jest.globals = { ...jest.globals } as any) // we need to remove some stuff from jest config // this which does not depend on config jest.name = undefined as any jest.cacheDirectory = undefined as any - // we do not need this since its normalized version is in tsJest - delete globals['ts-jest'] this._transformCfgStr = new JsonableValue({ digest: configSet.tsJestDigest, babel: configSet.babelConfig, @@ -84,11 +93,14 @@ export class TsJestTransformer implements Transformer { raw: configSet.parsedTsConfig.raw, }, }).serialized + this._compiler = new TsJestCompiler(configSet) TsJestTransformer._cachedConfigSets.push({ jestConfig: new JsonableValue(jestConfig), configSet, transformerCfgStr: this._transformCfgStr, + compiler: this._compiler, }) + this._getFsCachedResolvedModules(configSet) } } @@ -104,10 +116,10 @@ export class TsJestTransformer implements Transformer { jestConfig: Config.ProjectConfig, transformOptions?: TransformOptions, ): TransformedSource | string { - this.logger.debug({ fileName: filePath, transformOptions }, 'processing', filePath) + this._logger.debug({ fileName: filePath, transformOptions }, 'processing', filePath) let result: string | TransformedSource - const configs = this.configsFor(jestConfig) + const configs = this._configsFor(jestConfig) const { hooks } = configs const shouldStringifyContent = configs.shouldStringifyContent(filePath) const babelJest = shouldStringifyContent ? undefined : configs.babelJestTransformer @@ -122,28 +134,25 @@ export class TsJestTransformer implements Transformer { result = '' } else if (!configs.parsedTsConfig.options.allowJs && isJsFile) { // we've got a '.js' but the compiler option `allowJs` is not set or set to false - this.logger.warn({ fileName: filePath }, interpolate(Errors.GotJsFileButAllowJsFalse, { path: filePath })) + this._logger.warn({ fileName: filePath }, interpolate(Errors.GotJsFileButAllowJsFalse, { path: filePath })) result = fileContent } else if (isJsFile || isTsFile) { - if (!TsJestTransformer._compiler) { - TsJestTransformer._compiler = new TsJestCompiler(configs) - } // transpile TS code (source maps are included) - result = TsJestTransformer._compiler.getCompiledOutput(fileContent, filePath) + result = this._compiler.getCompiledOutput(fileContent, filePath) } else { // we should not get called for files with other extension than js[x], ts[x] and d.ts, // TypeScript will bail if we try to compile, and if it was to call babel, users can // define the transform value with `babel-jest` for this extension instead const message = babelJest ? Errors.GotUnknownFileTypeWithBabel : Errors.GotUnknownFileTypeWithoutBabel - this.logger.warn({ fileName: filePath }, interpolate(message, { path: filePath })) + this._logger.warn({ fileName: filePath }, interpolate(message, { path: filePath })) result = fileContent } // calling babel-jest transformer if (babelJest) { - this.logger.debug({ fileName: filePath }, 'calling babel-jest processor') + this._logger.debug({ fileName: filePath }, 'calling babel-jest processor') // do not instrument here, jest will do it anyway afterwards result = babelJest.process(result, filePath, jestConfig, { ...transformOptions, instrument: false }) @@ -151,7 +160,7 @@ export class TsJestTransformer implements Transformer { // allows hooks (useful for internal testing) /* istanbul ignore next (cover by e2e) */ if (hooks.afterProcess) { - this.logger.debug({ fileName: filePath, hookName: 'afterProcess' }, 'calling afterProcess hook') + this._logger.debug({ fileName: filePath, hookName: 'afterProcess' }, 'calling afterProcess hook') const newResult = hooks.afterProcess([fileContent, filePath, jestConfig, transformOptions], result) if (newResult !== undefined) { @@ -175,23 +184,88 @@ export class TsJestTransformer implements Transformer { _jestConfigStr: string, transformOptions: CacheKeyOptions, ): string { - const configs = this.configsFor(transformOptions.config) + const configs = this._configsFor(transformOptions.config) - this.logger.debug({ fileName: filePath, transformOptions }, 'computing cache key for', filePath) + this._logger.debug({ fileName: filePath, transformOptions }, 'computing cache key for', filePath) // we do not instrument, ensure it is false all the time const { instrument = false, rootDir = configs.rootDir } = transformOptions - - return sha1( + const constructingCacheKeyElements = [ this._transformCfgStr, - '\x00', + CACHE_KEY_EL_SEPARATOR, rootDir, - '\x00', + CACHE_KEY_EL_SEPARATOR, `instrument:${instrument ? 'on' : 'off'}`, - '\x00', + CACHE_KEY_EL_SEPARATOR, fileContent, - '\x00', + CACHE_KEY_EL_SEPARATOR, filePath, - ) + ] + if (!configs.isolatedModules) { + let resolvedModuleNames: string[] + if (this._depGraphs.get(filePath)?.fileContent === fileContent) { + this._logger.debug( + { fileName: filePath, transformOptions }, + 'getting resolved modules from disk caching or memory caching for', + filePath, + ) + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + resolvedModuleNames = this._depGraphs + .get(filePath)! + .resolveModuleNames.filter((moduleName) => existsSync(moduleName)) + } else { + this._logger.debug( + { fileName: filePath, transformOptions }, + 'getting resolved modules from TypeScript API for', + filePath, + ) + + const resolvedModuleMap = this._compiler.getResolvedModulesMap(fileContent, filePath) + resolvedModuleNames = resolvedModuleMap + ? [...resolvedModuleMap.values()] + .filter((resolvedModule) => resolvedModule !== undefined) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + .map((resolveModule) => resolveModule!.resolvedFileName) + : [] + this._depGraphs.set(filePath, { + fileContent, + resolveModuleNames: resolvedModuleNames, + }) + /* istanbul ignore next */ + if (this._tsResolvedModulesCachePath) { + // Cache resolved modules to disk so next run can reuse it + void (async () => { + // eslint-disable-next-line @typescript-eslint/await-thenable + await writeFile(this._tsResolvedModulesCachePath as string, stringify([...this._depGraphs]), () => {}) + })() + } + } + resolvedModuleNames.forEach((moduleName) => { + constructingCacheKeyElements.push( + CACHE_KEY_EL_SEPARATOR, + moduleName, + CACHE_KEY_EL_SEPARATOR, + statSync(moduleName).mtimeMs.toString(), + ) + }) + } + + return sha1(...constructingCacheKeyElements) + } + + protected _getFsCachedResolvedModules(configSet: ConfigSet): void { + if (!configSet.isolatedModules) { + const cacheDir = configSet.tsCacheDir + if (cacheDir) { + // Make sure the cache directory exists before continuing. + mkdirp.sync(cacheDir) + this._tsResolvedModulesCachePath = join(cacheDir, sha1('ts-jest-resolved-modules', CACHE_KEY_EL_SEPARATOR)) + try { + const cachedTSResolvedModules = readFileSync(this._tsResolvedModulesCachePath, 'utf-8') + this._depGraphs = new Map(parse(cachedTSResolvedModules)) + } catch (e) {} + } + } } } diff --git a/src/types.ts b/src/types.ts index 92566f5083..908f5b7967 100644 --- a/src/types.ts +++ b/src/types.ts @@ -172,7 +172,10 @@ export interface TsJestConfig { stringifyContentPathRegex: string | undefined } +export type ResolvedModulesMap = Map | undefined + export interface CompilerInstance { + getResolvedModulesMap(fileContent: string, fileName: string): ResolvedModulesMap getCompiledOutput(fileContent: string, fileName: string): string } /**