From 931b504836c13218f81f43c0e0211bec3535ac30 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 11 Nov 2021 14:21:07 -0800 Subject: [PATCH 1/4] Revert "Fix RWC missing file detection (#46673)" This reverts commit 4a065f524c1f843df297f6ee01313c6bc31d4f65. --- src/harness/harnessIO.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/harness/harnessIO.ts b/src/harness/harnessIO.ts index 0f1647b729427..bf9f45d8826b4 100644 --- a/src/harness/harnessIO.ts +++ b/src/harness/harnessIO.ts @@ -1438,15 +1438,14 @@ namespace Harness { } const referenceDir = referencePath(relativeFileBase, opts && opts.Baselinefolder, opts && opts.Subfolder); - let existing = IO.readDirectory(referenceDir, referencedExtensions || [extension]); // always an _absolute_ path + let existing = IO.readDirectory(referenceDir, referencedExtensions || [extension]); if (extension === ".ts" || referencedExtensions && referencedExtensions.indexOf(".ts") > -1 && referencedExtensions.indexOf(".d.ts") === -1) { // special-case and filter .d.ts out of .ts results existing = existing.filter(f => !ts.endsWith(f, ".d.ts")); } const missing: string[] = []; - const absoluteTestDir = `${process.cwd()}/${referenceDir}`; for (const name of existing) { - const localCopy = name.substring(absoluteTestDir.length - relativeFileBase.length); + const localCopy = name.substring(referenceDir.length - relativeFileBase.length); if (!writtenFiles.has(localCopy)) { missing.push(localCopy); } From afef282bb35200561913dd482d3e05f699e7b455 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 11 Nov 2021 14:21:33 -0800 Subject: [PATCH 2/4] Revert "Pass absolute path to directoryExists (#46086)" This reverts commit 55b4928e820c0a16698eff41ba08bfe550a75604. --- src/compiler/utilities.ts | 28 ++++++++++++------------ src/testRunner/unittests/publicApi.ts | 31 --------------------------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts index aa3e8e0ee92b7..ea71b61acc8ff 100644 --- a/src/compiler/utilities.ts +++ b/src/compiler/utilities.ts @@ -6629,7 +6629,7 @@ namespace ts { includeFilePattern: getRegularExpressionForWildcard(includes, absolutePath, "files"), includeDirectoryPattern: getRegularExpressionForWildcard(includes, absolutePath, "directories"), excludePattern: getRegularExpressionForWildcard(excludes, absolutePath, "exclude"), - basePaths: getBasePaths(absolutePath, includes, useCaseSensitiveFileNames) + basePaths: getBasePaths(path, includes, useCaseSensitiveFileNames) }; } @@ -6653,22 +6653,22 @@ namespace ts { const results: string[][] = includeFileRegexes ? includeFileRegexes.map(() => []) : [[]]; const visited = new Map(); const toCanonical = createGetCanonicalFileName(useCaseSensitiveFileNames); - for (const absoluteBasePath of patterns.basePaths) { - if (directoryExists(absoluteBasePath)) { - visitDirectory(absoluteBasePath, depth); + for (const basePath of patterns.basePaths) { + if (directoryExists(basePath)) { + visitDirectory(basePath, combinePaths(currentDirectory, basePath), depth); } } return flatten(results); - function visitDirectory(absolutePath: string, depth: number | undefined) { + function visitDirectory(path: string, absolutePath: string, depth: number | undefined) { const canonicalPath = toCanonical(realpath(absolutePath)); if (visited.has(canonicalPath)) return; visited.set(canonicalPath, true); - const { files, directories } = getFileSystemEntries(absolutePath); + const { files, directories } = getFileSystemEntries(path); for (const current of sort(files, compareStringsCaseSensitive)) { - const name = combinePaths(absolutePath, current); + const name = combinePaths(path, current); const absoluteName = combinePaths(absolutePath, current); if (extensions && !fileExtensionIsOneOf(name, extensions)) continue; if (excludeRegex && excludeRegex.test(absoluteName)) continue; @@ -6691,10 +6691,11 @@ namespace ts { } for (const current of sort(directories, compareStringsCaseSensitive)) { + const name = combinePaths(path, current); const absoluteName = combinePaths(absolutePath, current); if ((!includeDirectoryRegex || includeDirectoryRegex.test(absoluteName)) && (!excludeRegex || !excludeRegex.test(absoluteName))) { - visitDirectory(absoluteName, depth); + visitDirectory(name, absoluteName, depth); } } } @@ -6702,11 +6703,10 @@ namespace ts { /** * Computes the unique non-wildcard base paths amongst the provided include patterns. - * @returns Absolute directory paths */ - function getBasePaths(absoluteTsconfigPath: string, includes: readonly string[] | undefined, useCaseSensitiveFileNames: boolean): string[] { + function getBasePaths(path: string, includes: readonly string[] | undefined, useCaseSensitiveFileNames: boolean): string[] { // Storage for our results in the form of literal paths (e.g. the paths as written by the user). - const basePaths: string[] = [absoluteTsconfigPath]; + const basePaths: string[] = [path]; if (includes) { // Storage for literal base paths amongst the include patterns. @@ -6714,9 +6714,9 @@ namespace ts { for (const include of includes) { // We also need to check the relative paths by converting them to absolute and normalizing // in case they escape the base path (e.g "..\somedirectory") - const absoluteIncludePath: string = isRootedDiskPath(include) ? include : normalizePath(combinePaths(absoluteTsconfigPath, include)); + const absolute: string = isRootedDiskPath(include) ? include : normalizePath(combinePaths(path, include)); // Append the literal and canonical candidate base paths. - includeBasePaths.push(getIncludeBasePath(absoluteIncludePath)); + includeBasePaths.push(getIncludeBasePath(absolute)); } // Sort the offsets array using either the literal or canonical path representations. @@ -6725,7 +6725,7 @@ namespace ts { // Iterate over each include base path and include unique base paths that are not a // subpath of an existing base path for (const includeBasePath of includeBasePaths) { - if (every(basePaths, basePath => !containsPath(basePath, includeBasePath, absoluteTsconfigPath, !useCaseSensitiveFileNames))) { + if (every(basePaths, basePath => !containsPath(basePath, includeBasePath, path, !useCaseSensitiveFileNames))) { basePaths.push(includeBasePath); } } diff --git a/src/testRunner/unittests/publicApi.ts b/src/testRunner/unittests/publicApi.ts index e741e902b4e31..883b979f9c72e 100644 --- a/src/testRunner/unittests/publicApi.ts +++ b/src/testRunner/unittests/publicApi.ts @@ -182,34 +182,3 @@ describe("unittests:: Public APIs:: getChild* methods on EndOfFileToken with JSD assert.equal(endOfFileToken.getChildCount(), 1); assert.notEqual(endOfFileToken.getChildAt(0), /*expected*/ undefined); }); - -describe("unittests:: Public APIs:: sys", () => { - it("readDirectory", () => { - // #45990, testing passing a non-absolute path - // `sys.readDirectory` is just `matchFiles` plugged into the real FS - const read = ts.matchFiles( - /*path*/ "", - /*extensions*/ [".ts", ".tsx"], - /*excludes*/ ["node_modules", "dist"], - /*includes*/ ["**/*"], - /*useCaseSensitiveFileNames*/ true, - /*currentDirectory*/ "/", - /*depth*/ undefined, - /*getFileSystemEntries*/ path => { - switch (path) { - case "/": return { directories: [], files: ["file.ts"] }; - default: return { directories: [], files: [] }; - } - }, - /*realpath*/ ts.identity, - /*directoryExists*/ path => { - switch (path) { - case "/": return true; - default: return false; - } - } - ); - - assert.deepEqual(read, ["/file.ts"]); - }); -}); From f1a20b34a2698469c98ad603bda04acaf198e245 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 11 Nov 2021 14:22:02 -0800 Subject: [PATCH 3/4] Revert "Reduce exceptions (#44710)" This reverts commit c0d5c29080e242f85a55e52e42f61065042d1730. --- src/compiler/sys.ts | 12 ++---------- src/compiler/utilities.ts | 6 ++---- src/compiler/watchUtilities.ts | 2 +- src/harness/fakesHosts.ts | 2 +- src/harness/virtualFileSystemWithWatch.ts | 2 +- 5 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/compiler/sys.ts b/src/compiler/sys.ts index a5330d807cddc..da38d389f212f 100644 --- a/src/compiler/sys.ts +++ b/src/compiler/sys.ts @@ -1266,7 +1266,6 @@ namespace ts { let activeSession: import("inspector").Session | "stopping" | undefined; let profilePath = "./profile.cpuprofile"; - let hitSystemWatcherLimit = false; const Buffer: { new (input: string, encoding?: string): any; @@ -1620,12 +1619,6 @@ namespace ts { options = { persistent: true }; } } - - if (hitSystemWatcherLimit) { - sysLog(`sysLog:: ${fileOrDirectory}:: Defaulting to fsWatchFile`); - return watchPresentFileSystemEntryWithFsWatchFile(); - } - try { const presentWatcher = _fs.watch( fileOrDirectory, @@ -1642,8 +1635,6 @@ namespace ts { // Catch the exception and use polling instead // Eg. on linux the number of watches are limited and one could easily exhaust watches and the exception ENOSPC is thrown when creating watcher at that point // so instead of throwing error, use fs.watchFile - hitSystemWatcherLimit ||= e.code === "ENOSPC"; - sysLog(`sysLog:: ${fileOrDirectory}:: Changing to fsWatchFile`); return watchPresentFileSystemEntryWithFsWatchFile(); } } @@ -1665,6 +1656,7 @@ namespace ts { * Eg. on linux the number of watches are limited and one could easily exhaust watches and the exception ENOSPC is thrown when creating watcher at that point */ function watchPresentFileSystemEntryWithFsWatchFile(): FileWatcher { + sysLog(`sysLog:: ${fileOrDirectory}:: Changing to fsWatchFile`); return watchFile( fileOrDirectory, createFileWatcherCallback(callback), @@ -1804,7 +1796,7 @@ namespace ts { } function readDirectory(path: string, extensions?: readonly string[], excludes?: readonly string[], includes?: readonly string[], depth?: number): string[] { - return matchFiles(path, extensions, excludes, includes, useCaseSensitiveFileNames, process.cwd(), depth, getAccessibleFileSystemEntries, realpath, directoryExists); + return matchFiles(path, extensions, excludes, includes, useCaseSensitiveFileNames, process.cwd(), depth, getAccessibleFileSystemEntries, realpath); } function fileSystemEntryExists(path: string, entryKind: FileSystemEntryKind): boolean { diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts index ea71b61acc8ff..9c706c6f61814 100644 --- a/src/compiler/utilities.ts +++ b/src/compiler/utilities.ts @@ -6638,7 +6638,7 @@ namespace ts { } /** @param path directory of the tsconfig.json */ - export function matchFiles(path: string, extensions: readonly string[] | undefined, excludes: readonly string[] | undefined, includes: readonly string[] | undefined, useCaseSensitiveFileNames: boolean, currentDirectory: string, depth: number | undefined, getFileSystemEntries: (path: string) => FileSystemEntries, realpath: (path: string) => string, directoryExists: (path: string) => boolean): string[] { + export function matchFiles(path: string, extensions: readonly string[] | undefined, excludes: readonly string[] | undefined, includes: readonly string[] | undefined, useCaseSensitiveFileNames: boolean, currentDirectory: string, depth: number | undefined, getFileSystemEntries: (path: string) => FileSystemEntries, realpath: (path: string) => string): string[] { path = normalizePath(path); currentDirectory = normalizePath(currentDirectory); @@ -6654,9 +6654,7 @@ namespace ts { const visited = new Map(); const toCanonical = createGetCanonicalFileName(useCaseSensitiveFileNames); for (const basePath of patterns.basePaths) { - if (directoryExists(basePath)) { - visitDirectory(basePath, combinePaths(currentDirectory, basePath), depth); - } + visitDirectory(basePath, combinePaths(currentDirectory, basePath), depth); } return flatten(results); diff --git a/src/compiler/watchUtilities.ts b/src/compiler/watchUtilities.ts index c6b9df2c3e3b5..adbf9f1287b1b 100644 --- a/src/compiler/watchUtilities.ts +++ b/src/compiler/watchUtilities.ts @@ -184,7 +184,7 @@ namespace ts { const rootResult = tryReadDirectory(rootDir, rootDirPath); let rootSymLinkResult: FileSystemEntries | undefined; if (rootResult !== undefined) { - return matchFiles(rootDir, extensions, excludes, includes, useCaseSensitiveFileNames, currentDirectory, depth, getFileSystemEntries, realpath, directoryExists); + return matchFiles(rootDir, extensions, excludes, includes, useCaseSensitiveFileNames, currentDirectory, depth, getFileSystemEntries, realpath); } return host.readDirectory!(rootDir, extensions, excludes, includes, depth); diff --git a/src/harness/fakesHosts.ts b/src/harness/fakesHosts.ts index 898cfc048c32e..58b29a4c7ef98 100644 --- a/src/harness/fakesHosts.ts +++ b/src/harness/fakesHosts.ts @@ -95,7 +95,7 @@ namespace fakes { } public readDirectory(path: string, extensions?: readonly string[], exclude?: readonly string[], include?: readonly string[], depth?: number): string[] { - return ts.matchFiles(path, extensions, exclude, include, this.useCaseSensitiveFileNames, this.getCurrentDirectory(), depth, path => this.getAccessibleFileSystemEntries(path), path => this.realpath(path), path => this.directoryExists(path)); + return ts.matchFiles(path, extensions, exclude, include, this.useCaseSensitiveFileNames, this.getCurrentDirectory(), depth, path => this.getAccessibleFileSystemEntries(path), path => this.realpath(path)); } public getAccessibleFileSystemEntries(path: string): ts.FileSystemEntries { diff --git a/src/harness/virtualFileSystemWithWatch.ts b/src/harness/virtualFileSystemWithWatch.ts index 77e3741d1fce8..62c6cd042ef93 100644 --- a/src/harness/virtualFileSystemWithWatch.ts +++ b/src/harness/virtualFileSystemWithWatch.ts @@ -922,7 +922,7 @@ interface Array { length: number; [n: number]: T; }` }); } return { directories, files }; - }, path => this.realpath(path), path => this.directoryExists(path)); + }, path => this.realpath(path)); } createHash(s: string): string { From 56842cd3bc42362576befc4124887b89e7006515 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 11 Nov 2021 14:57:15 -0800 Subject: [PATCH 4/4] Add back system watcher limit --- src/compiler/sys.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/compiler/sys.ts b/src/compiler/sys.ts index da38d389f212f..b41e89e991567 100644 --- a/src/compiler/sys.ts +++ b/src/compiler/sys.ts @@ -1266,6 +1266,7 @@ namespace ts { let activeSession: import("inspector").Session | "stopping" | undefined; let profilePath = "./profile.cpuprofile"; + let hitSystemWatcherLimit = false; const Buffer: { new (input: string, encoding?: string): any; @@ -1619,6 +1620,11 @@ namespace ts { options = { persistent: true }; } } + + if (hitSystemWatcherLimit) { + sysLog(`sysLog:: ${fileOrDirectory}:: Defaulting to fsWatchFile`); + return watchPresentFileSystemEntryWithFsWatchFile(); + } try { const presentWatcher = _fs.watch( fileOrDirectory, @@ -1635,6 +1641,8 @@ namespace ts { // Catch the exception and use polling instead // Eg. on linux the number of watches are limited and one could easily exhaust watches and the exception ENOSPC is thrown when creating watcher at that point // so instead of throwing error, use fs.watchFile + hitSystemWatcherLimit ||= e.code === "ENOSPC"; + sysLog(`sysLog:: ${fileOrDirectory}:: Changing to fsWatchFile`); return watchPresentFileSystemEntryWithFsWatchFile(); } } @@ -1656,7 +1664,6 @@ namespace ts { * Eg. on linux the number of watches are limited and one could easily exhaust watches and the exception ENOSPC is thrown when creating watcher at that point */ function watchPresentFileSystemEntryWithFsWatchFile(): FileWatcher { - sysLog(`sysLog:: ${fileOrDirectory}:: Changing to fsWatchFile`); return watchFile( fileOrDirectory, createFileWatcherCallback(callback),