From 9ed1fac84d81755ffd47e9ee8eb48a06b94d42f4 Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Wed, 9 Jun 2021 23:18:31 +0400
Subject: [PATCH 1/7] feat(typescript-estree): detect single run and create
 programs for projects up front

---
 .../src/eslint-utils/RuleTester.ts            |   2 +-
 packages/parser/src/index.ts                  |   2 +-
 .../src/create-program/createWatchProgram.ts  |   4 +-
 packages/typescript-estree/src/index.ts       |   2 +-
 .../typescript-estree/src/parser-options.ts   |   1 +
 packages/typescript-estree/src/parser.ts      |  90 ++++++++-
 .../typescript-estree/tests/lib/parse.test.ts |   2 +-
 .../tests/lib/persistentParse.test.ts         |   4 +-
 .../tests/lib/semanticInfo-singleRun.test.ts  | 175 ++++++++++++++++++
 .../tests/lib/semanticInfo.test.ts            |   6 +-
 10 files changed, 275 insertions(+), 13 deletions(-)
 create mode 100644 packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts

diff --git a/packages/experimental-utils/src/eslint-utils/RuleTester.ts b/packages/experimental-utils/src/eslint-utils/RuleTester.ts
index a3210162e98..7479684393c 100644
--- a/packages/experimental-utils/src/eslint-utils/RuleTester.ts
+++ b/packages/experimental-utils/src/eslint-utils/RuleTester.ts
@@ -33,7 +33,7 @@ class RuleTester extends TSESLint.RuleTester {
         // instead of creating a hard dependency, just use a soft require
         // a bit weird, but if they're using this tooling, it'll be installed
         // eslint-disable-next-line @typescript-eslint/no-unsafe-call
-        require(parser).clearCaches();
+        require(parser).clearWatchCaches();
       } catch {
         // ignored
       }
diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts
index c1bb82bf593..1ae825f7537 100644
--- a/packages/parser/src/index.ts
+++ b/packages/parser/src/index.ts
@@ -1,7 +1,7 @@
 export { parse, parseForESLint, ParserOptions } from './parser';
 export {
   ParserServices,
-  clearCaches,
+  clearWatchCaches,
   createProgram,
 } from '@typescript-eslint/typescript-estree';
 
diff --git a/packages/typescript-estree/src/create-program/createWatchProgram.ts b/packages/typescript-estree/src/create-program/createWatchProgram.ts
index e2bf060050c..f10f2a2295f 100644
--- a/packages/typescript-estree/src/create-program/createWatchProgram.ts
+++ b/packages/typescript-estree/src/create-program/createWatchProgram.ts
@@ -50,7 +50,7 @@ const parsedFilesSeenHash = new Map<CanonicalPath, string>();
  * Clear all of the parser caches.
  * This should only be used in testing to ensure the parser is clean between tests.
  */
-function clearCaches(): void {
+function clearWatchCaches(): void {
   knownWatchProgramMap.clear();
   fileWatchCallbackTrackingMap.clear();
   folderWatchCallbackTrackingMap.clear();
@@ -530,4 +530,4 @@ function maybeInvalidateProgram(
   return null;
 }
 
-export { clearCaches, createWatchProgram, getProgramsForProjects };
+export { clearWatchCaches, createWatchProgram, getProgramsForProjects };
diff --git a/packages/typescript-estree/src/index.ts b/packages/typescript-estree/src/index.ts
index 3345d4d46ce..d00e8a1d953 100644
--- a/packages/typescript-estree/src/index.ts
+++ b/packages/typescript-estree/src/index.ts
@@ -2,7 +2,7 @@ export * from './parser';
 export { ParserServices, TSESTreeOptions } from './parser-options';
 export { simpleTraverse } from './simple-traverse';
 export * from './ts-estree';
-export { clearCaches } from './create-program/createWatchProgram';
+export { clearWatchCaches } from './create-program/createWatchProgram';
 export { createProgramFromConfigFile as createProgram } from './create-program/useProvidedPrograms';
 
 // re-export for backwards-compat
diff --git a/packages/typescript-estree/src/parser-options.ts b/packages/typescript-estree/src/parser-options.ts
index 9e33627ac9a..99ca7fa6504 100644
--- a/packages/typescript-estree/src/parser-options.ts
+++ b/packages/typescript-estree/src/parser-options.ts
@@ -18,6 +18,7 @@ export interface Extra {
   filePath: string;
   jsx: boolean;
   loc: boolean;
+  singleRun: boolean;
   log: (message: string) => void;
   preserveNodeMaps?: boolean;
   programs: null | Program[];
diff --git a/packages/typescript-estree/src/parser.ts b/packages/typescript-estree/src/parser.ts
index 7fe00818959..8c9301b8ed0 100644
--- a/packages/typescript-estree/src/parser.ts
+++ b/packages/typescript-estree/src/parser.ts
@@ -2,6 +2,7 @@ import debug from 'debug';
 import { sync as globSync } from 'globby';
 import isGlob from 'is-glob';
 import semver from 'semver';
+import { normalize } from 'path';
 import * as ts from 'typescript';
 import { astConverter } from './ast-converter';
 import { convertError } from './convert';
@@ -19,7 +20,10 @@ import {
   getCanonicalFileName,
 } from './create-program/shared';
 import { Program } from 'typescript';
-import { useProvidedPrograms } from './create-program/useProvidedPrograms';
+import {
+  createProgramFromConfigFile,
+  useProvidedPrograms,
+} from './create-program/useProvidedPrograms';
 
 const log = debug('typescript-eslint:typescript-estree:parser');
 
@@ -44,6 +48,16 @@ const isRunningSupportedTypeScriptVersion = semver.satisfies(
 let extra: Extra;
 let warnedAboutTSVersion = false;
 
+/**
+ * Cache existing programs for the single run use-case.
+ *
+ * clearProgramCache() is only intended to be used in testing to ensure the parser is clean between tests.
+ */
+const existingPrograms = new Map<CanonicalPath, ts.Program>();
+function clearProgramCache(): void {
+  existingPrograms.clear();
+}
+
 function enforceString(code: unknown): string {
   /**
    * Ensure the source code is a string
@@ -118,6 +132,11 @@ function resetExtra(): void {
     tokens: null,
     tsconfigRootDir: process.cwd(),
     useJSXTextNode: false,
+    /**
+     * Unless we can reliably infer otherwise, we default to assuming that this run could be part
+     * of a long-running session (e.g. in an IDE) and watch programs will therefore be required
+     */
+    singleRun: false,
   };
 }
 
@@ -347,6 +366,41 @@ function warnAboutTSVersion(): void {
   }
 }
 
+/**
+ * ESLint (and therefore typescript-eslint) is used in both "single run"/one-time contexts,
+ * such as an ESLint CLI invocation, and long-running sessions (such as continuous feedback
+ * on a file in an IDE).
+ *
+ * When typescript-eslint handles TypeScript Program management behind the scenes, this distinction
+ * is important because there is significant overhead to managing the so called Watch Programs
+ * needed for the long-running use-case. We therefore use the following logic to figure out which
+ * of these contexts applies to the current execution.
+ */
+function inferSingleRun(): void {
+  // Allow users to explicitly inform us of their intent to perform a single run (or not) with TSESTREE_SINGLE_RUN
+  if (process.env.TSESTREE_SINGLE_RUN === 'false') {
+    extra.singleRun = false;
+    return;
+  }
+
+  if (
+    process.env.TSESTREE_SINGLE_RUN === 'true' ||
+    // Default to single runs for CI processes. CI=true is set by most CI providers by default.
+    process.env.CI === 'true' ||
+    // This will be true for invocations such as `npx eslint ...` and `./node_modules/.bin/eslint ...`
+    process.argv[1].endsWith(normalize('node_modules/.bin/eslint'))
+  ) {
+    extra.singleRun = true;
+    return;
+  }
+
+  /**
+   * We default to assuming that this run could be part of a long-running session (e.g. in an IDE)
+   * and watch programs will therefore be required
+   */
+  extra.singleRun = false;
+}
+
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 interface EmptyObject {}
 type AST<T extends TSESTreeOptions> = TSESTree.Program &
@@ -408,6 +462,11 @@ function parseWithNodeMapsInternal<T extends TSESTreeOptions = TSESTreeOptions>(
    */
   warnAboutTSVersion();
 
+  /**
+   * Figure out whether this is a single run or part of a long-running process
+   */
+  inferSingleRun();
+
   /**
    * Create a ts.SourceFile directly, no ts.Program is needed for a simple
    * parse
@@ -468,7 +527,33 @@ function parseAndGenerateServices<T extends TSESTreeOptions = TSESTreeOptions>(
   warnAboutTSVersion();
 
   /**
-   * Generate a full ts.Program or offer provided instance in order to be able to provide parser services, such as type-checking
+   * Figure out whether this is a single run or part of a long-running process
+   */
+  inferSingleRun();
+
+  /**
+   * If this is a single run in which the user has not provided any existing programs but there
+   * are programs which need to be created from the provided "project" option,
+   * create the programs once ahead of time and avoid watch programs
+   */
+  if (extra.singleRun && !extra.programs && extra.projects?.length > 0) {
+    extra.programs = extra.projects.map(configFile => {
+      const existingProgram = existingPrograms.get(configFile);
+      if (existingProgram) {
+        return existingProgram;
+      }
+      log(
+        'Detected single-run/CLI usage, creating Program once ahead of time for project: %s',
+        configFile,
+      );
+      const newProgram = createProgramFromConfigFile(configFile);
+      existingPrograms.set(configFile, newProgram);
+      return newProgram;
+    });
+  }
+
+  /**
+   * Generate a full ts.Program or offer provided instances in order to be able to provide parser services, such as type-checking
    */
   const shouldProvideParserServices =
     extra.programs != null || (extra.projects && extra.projects.length > 0);
@@ -519,4 +604,5 @@ export {
   parseWithNodeMaps,
   ParseAndGenerateServicesResult,
   ParseWithNodeMapsResult,
+  clearProgramCache,
 };
diff --git a/packages/typescript-estree/tests/lib/parse.test.ts b/packages/typescript-estree/tests/lib/parse.test.ts
index 3e312973700..cfd4db39acf 100644
--- a/packages/typescript-estree/tests/lib/parse.test.ts
+++ b/packages/typescript-estree/tests/lib/parse.test.ts
@@ -632,7 +632,7 @@ describe('parseAndGenerateServices', () => {
 
   describe('projectFolderIgnoreList', () => {
     beforeEach(() => {
-      parser.clearCaches();
+      parser.clearWatchCaches();
     });
 
     const PROJECT_DIR = resolve(FIXTURES_DIR, '../projectFolderIgnoreList');
diff --git a/packages/typescript-estree/tests/lib/persistentParse.test.ts b/packages/typescript-estree/tests/lib/persistentParse.test.ts
index 7e751ed4248..5de5ed7a471 100644
--- a/packages/typescript-estree/tests/lib/persistentParse.test.ts
+++ b/packages/typescript-estree/tests/lib/persistentParse.test.ts
@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import tmp from 'tmp';
-import { clearCaches, parseAndGenerateServices } from '../../src';
+import { clearWatchCaches, parseAndGenerateServices } from '../../src';
 
 const CONTENTS = {
   foo: 'console.log("foo")',
@@ -17,7 +17,7 @@ const cwdCopy = process.cwd();
 const tmpDirs = new Set<tmp.DirResult>();
 afterEach(() => {
   // stop watching the files and folders
-  clearCaches();
+  clearWatchCaches();
 
   // clean up the temporary files and folders
   tmpDirs.forEach(t => t.removeCallback());
diff --git a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
new file mode 100644
index 00000000000..bfd4a85da13
--- /dev/null
+++ b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
@@ -0,0 +1,175 @@
+import glob from 'glob';
+import * as path from 'path';
+import { clearProgramCache, parseAndGenerateServices } from '../../src';
+
+const mockProgram = {
+  getSourceFile(): void {
+    return;
+  },
+};
+
+jest.mock('../../src/ast-converter', () => {
+  return {
+    astConverter(): unknown {
+      return { estree: {}, astMaps: {} };
+    },
+  };
+});
+
+jest.mock('../../src/create-program/useProvidedPrograms.ts', () => {
+  return {
+    ...jest.requireActual('../../src/create-program/useProvidedPrograms.ts'),
+    useProvidedPrograms: jest.fn(() => {
+      return {
+        ast: {
+          parseDiagnostics: [],
+        },
+        program: mockProgram,
+      };
+    }),
+    createProgramFromConfigFile: jest.fn(() => mockProgram),
+  };
+});
+
+const {
+  createProgramFromConfigFile,
+} = require('../../src/create-program/useProvidedPrograms');
+
+const FIXTURES_DIR = './tests/fixtures/semanticInfo';
+const testFiles = glob.sync(`**/*.src.ts`, {
+  cwd: FIXTURES_DIR,
+});
+
+const code = 'const foo = 5;';
+const tsconfigs = ['./tsconfig.json', './badTSConfig/tsconfig.json'];
+const options = {
+  filePath: testFiles[0],
+  tsconfigRootDir: path.join(process.cwd(), FIXTURES_DIR),
+  loggerFn: false,
+  project: tsconfigs,
+} as const;
+
+const resolvedProject = (p: string): string =>
+  path.resolve(path.join(process.cwd(), FIXTURES_DIR), p).toLowerCase();
+
+describe('semanticInfo - singleRun', () => {
+  beforeEach(() => {
+    // ensure caches are clean for each test
+    clearProgramCache();
+    // ensure invocations of mock are clean for each test
+    (createProgramFromConfigFile as jest.Mock).mockClear();
+  });
+
+  it('should not create any programs ahead of time by default when there is no way to infer singleRun=true', () => {
+    /**
+     * Nothing to indicate it is a single run, so createProgramFromConfigFile should
+     * never be called
+     */
+    parseAndGenerateServices(code, options);
+    expect(createProgramFromConfigFile).not.toHaveBeenCalled();
+  });
+
+  it('should not create any programs ahead of time when when TSESTREE_SINGLE_RUN=false, even if other inferrence criteria apply', () => {
+    const originalTSESTreeSingleRun = process.env.TSESTREE_SINGLE_RUN;
+    process.env.TSESTREE_SINGLE_RUN = 'false';
+
+    // Normally CI=true would be used to infer singleRun=true, but TSESTREE_SINGLE_RUN is explicitly set to false
+    const originalEnvCI = process.env.CI;
+    process.env.CI = 'true';
+
+    parseAndGenerateServices(code, options);
+    expect(createProgramFromConfigFile).not.toHaveBeenCalled();
+
+    // Restore process data
+    process.env.TSESTREE_SINGLE_RUN = originalTSESTreeSingleRun;
+    process.env.CI = originalEnvCI;
+  });
+
+  it('should create all programs for provided "parserOptions.project" once ahead of time when TSESTREE_SINGLE_RUN=true', () => {
+    /**
+     * Single run because of explicit environment variable TSESTREE_SINGLE_RUN
+     */
+    const originalTSESTreeSingleRun = process.env.TSESTREE_SINGLE_RUN;
+    process.env.TSESTREE_SINGLE_RUN = 'true';
+
+    const resultProgram = parseAndGenerateServices(code, options).services
+      .program;
+    expect(resultProgram).toBe(mockProgram);
+
+    // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
+    parseAndGenerateServices(code, options);
+    // ...by asserting this was only called once per project
+    expect(createProgramFromConfigFile).toHaveBeenCalledTimes(tsconfigs.length);
+
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      1,
+      resolvedProject(tsconfigs[0]),
+    );
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      2,
+      resolvedProject(tsconfigs[1]),
+    );
+
+    // Restore process data
+    process.env.TSESTREE_SINGLE_RUN = originalTSESTreeSingleRun;
+  });
+
+  it('should create all programs for provided "parserOptions.project" once ahead of time when singleRun is inferred from CI=true', () => {
+    /**
+     * Single run because of CI=true (we need to make sure we respect the original value
+     * so that we won't interfere with our own usage of the variable)
+     */
+    const originalEnvCI = process.env.CI;
+    process.env.CI = 'true';
+
+    const resultProgram = parseAndGenerateServices(code, options).services
+      .program;
+    expect(resultProgram).toBe(mockProgram);
+
+    // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
+    parseAndGenerateServices(code, options);
+    // ...by asserting this was only called once per project
+    expect(createProgramFromConfigFile).toHaveBeenCalledTimes(tsconfigs.length);
+
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      1,
+      resolvedProject(tsconfigs[0]),
+    );
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      2,
+      resolvedProject(tsconfigs[1]),
+    );
+
+    // Restore process data
+    process.env.CI = originalEnvCI;
+  });
+
+  it('should create all programs for provided "parserOptions.project" once ahead of time when singleRun is inferred from process.argv', () => {
+    /**
+     * Single run because of process.argv
+     */
+    const originalProcessArgv = process.argv;
+    process.argv = ['', 'node_modules/.bin/eslint', ''];
+
+    const resultProgram = parseAndGenerateServices(code, options).services
+      .program;
+    expect(resultProgram).toBe(mockProgram);
+
+    // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
+    parseAndGenerateServices(code, options);
+    // ...by asserting this was only called once per project
+    expect(createProgramFromConfigFile).toHaveBeenCalledTimes(tsconfigs.length);
+
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      1,
+      resolvedProject(tsconfigs[0]),
+    );
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      2,
+      resolvedProject(tsconfigs[1]),
+    );
+
+    // Restore process data
+    process.argv = originalProcessArgv;
+  });
+});
diff --git a/packages/typescript-estree/tests/lib/semanticInfo.test.ts b/packages/typescript-estree/tests/lib/semanticInfo.test.ts
index 397fb4ddad4..7c448362b98 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo.test.ts
@@ -9,7 +9,7 @@ import {
   parseCodeAndGenerateServices,
 } from '../../tools/test-utils';
 import {
-  clearCaches,
+  clearWatchCaches,
   createProgram,
   parseAndGenerateServices,
   ParseAndGenerateServicesResult,
@@ -37,8 +37,8 @@ function createOptions(fileName: string): TSESTreeOptions & { cwd?: string } {
   };
 }
 
-// ensure tsconfig-parser caches are clean for each test
-beforeEach(() => clearCaches());
+// ensure tsconfig-parser watch caches are clean for each test
+beforeEach(() => clearWatchCaches());
 
 describe('semanticInfo', () => {
   // test all AST snapshots

From e24e93cdfa6e30a892306e6dc41719f1c09da187 Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 21:18:41 +0400
Subject: [PATCH 2/7] fix: apply PR feedback

---
 .eslintrc.js                                  |  1 +
 packages/typescript-estree/README.md          | 14 +++
 .../src/create-program/useProvidedPrograms.ts |  2 +-
 .../typescript-estree/src/parser-options.ts   | 20 +++-
 packages/typescript-estree/src/parser.ts      | 69 +++++++------
 .../tests/lib/semanticInfo-singleRun.test.ts  | 96 +++++++++++++++----
 6 files changed, 152 insertions(+), 50 deletions(-)

diff --git a/.eslintrc.js b/.eslintrc.js
index 41f25f55671..b39e5187bc7 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -25,6 +25,7 @@ module.exports = {
       './tests/integration/utils/jsconfig.json',
       './packages/*/tsconfig.json',
     ],
+    allowAutomaticSingleRunInference: true,
     tsconfigRootDir: __dirname,
     warnOnUnsupportedTypeScriptVersion: false,
     EXPERIMENTAL_useSourceOfProjectReferenceRedirect: false,
diff --git a/packages/typescript-estree/README.md b/packages/typescript-estree/README.md
index fca834e2209..31084150646 100644
--- a/packages/typescript-estree/README.md
+++ b/packages/typescript-estree/README.md
@@ -225,6 +225,20 @@ interface ParseAndGenerateServicesOptions extends ParseOptions {
    * it will not error, but will instead parse the file and its dependencies in a new program.
    */
   createDefaultProgram?: boolean;
+
+  /**
+   * ESLint (and therefore typescript-eslint) is used in both "single run"/one-time contexts,
+   * such as an ESLint CLI invocation, and long-running sessions (such as continuous feedback
+   * on a file in an IDE).
+   *
+   * When typescript-eslint handles TypeScript Program management behind the scenes, this distinction
+   * is important because there is significant overhead to managing the so called Watch Programs
+   * needed for the long-running use-case.
+   *
+   * When allowAutomaticSingleRunInference is enabled, we will use common heuristics to infer
+   * whether or not ESLint is being used as part of a single run.
+   */
+  allowAutomaticSingleRunInference?: boolean;
 }
 
 interface ParserServices {
diff --git a/packages/typescript-estree/src/create-program/useProvidedPrograms.ts b/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
index d807a0f4675..dd12f17461a 100644
--- a/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
+++ b/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
@@ -12,7 +12,7 @@ import {
 const log = debug('typescript-eslint:typescript-estree:useProvidedProgram');
 
 function useProvidedPrograms(
-  programInstances: ts.Program[],
+  programInstances: ts.Program[] | Iterable<ts.Program>,
   extra: Extra,
 ): ASTAndProgram | undefined {
   log(
diff --git a/packages/typescript-estree/src/parser-options.ts b/packages/typescript-estree/src/parser-options.ts
index 99ca7fa6504..0bde1d9176a 100644
--- a/packages/typescript-estree/src/parser-options.ts
+++ b/packages/typescript-estree/src/parser-options.ts
@@ -1,7 +1,7 @@
 import { DebugLevel } from '@typescript-eslint/types';
-import { Program } from 'typescript';
-import { TSESTree, TSNode, TSESTreeToTSNode, TSToken } from './ts-estree';
+import type { Program } from 'typescript';
 import { CanonicalPath } from './create-program/shared';
+import { TSESTree, TSESTreeToTSNode, TSNode, TSToken } from './ts-estree';
 
 type DebugModule = 'typescript-eslint' | 'eslint' | 'typescript';
 
@@ -21,7 +21,7 @@ export interface Extra {
   singleRun: boolean;
   log: (message: string) => void;
   preserveNodeMaps?: boolean;
-  programs: null | Program[];
+  programs: null | Iterable<Program>;
   projects: CanonicalPath[];
   range: boolean;
   strict: boolean;
@@ -188,6 +188,20 @@ interface ParseAndGenerateServicesOptions extends ParseOptions {
    * it will not error, but will instead parse the file and its dependencies in a new program.
    */
   createDefaultProgram?: boolean;
+
+  /**
+   * ESLint (and therefore typescript-eslint) is used in both "single run"/one-time contexts,
+   * such as an ESLint CLI invocation, and long-running sessions (such as continuous feedback
+   * on a file in an IDE).
+   *
+   * When typescript-eslint handles TypeScript Program management behind the scenes, this distinction
+   * is important because there is significant overhead to managing the so called Watch Programs
+   * needed for the long-running use-case.
+   *
+   * When allowAutomaticSingleRunInference is enabled, we will use common heuristics to infer
+   * whether or not ESLint is being used as part of a single run.
+   */
+  allowAutomaticSingleRunInference?: boolean;
 }
 
 export type TSESTreeOptions = ParseAndGenerateServicesOptions;
diff --git a/packages/typescript-estree/src/parser.ts b/packages/typescript-estree/src/parser.ts
index 8c9301b8ed0..904bb054fdf 100644
--- a/packages/typescript-estree/src/parser.ts
+++ b/packages/typescript-estree/src/parser.ts
@@ -19,7 +19,6 @@ import {
   ensureAbsolutePath,
   getCanonicalFileName,
 } from './create-program/shared';
-import { Program } from 'typescript';
 import {
   createProgramFromConfigFile,
   useProvidedPrograms,
@@ -71,20 +70,19 @@ function enforceString(code: unknown): string {
 
 /**
  * @param code The code of the file being linted
- * @param programInstances One or more existing programs to use
+ * @param programInstances One or more (potentially lazily constructed) existing programs to use
  * @param shouldProvideParserServices True if the program should be attempted to be calculated from provided tsconfig files
  * @param shouldCreateDefaultProgram True if the program should be created from compiler host
  * @returns Returns a source file and program corresponding to the linted code
  */
 function getProgramAndAST(
   code: string,
-  programInstances: Program[] | null,
+  programInstances: Iterable<ts.Program> | null,
   shouldProvideParserServices: boolean,
   shouldCreateDefaultProgram: boolean,
 ): ASTAndProgram {
   return (
-    (programInstances?.length &&
-      useProvidedPrograms(programInstances, extra)) ||
+    (programInstances && useProvidedPrograms(programInstances, extra)) ||
     (shouldProvideParserServices &&
       createProjectProgram(code, shouldCreateDefaultProgram, extra)) ||
     (shouldProvideParserServices &&
@@ -376,24 +374,30 @@ function warnAboutTSVersion(): void {
  * needed for the long-running use-case. We therefore use the following logic to figure out which
  * of these contexts applies to the current execution.
  */
-function inferSingleRun(): void {
+function inferSingleRun(options: TSESTreeOptions | undefined): void {
   // Allow users to explicitly inform us of their intent to perform a single run (or not) with TSESTREE_SINGLE_RUN
   if (process.env.TSESTREE_SINGLE_RUN === 'false') {
     extra.singleRun = false;
     return;
   }
-
-  if (
-    process.env.TSESTREE_SINGLE_RUN === 'true' ||
-    // Default to single runs for CI processes. CI=true is set by most CI providers by default.
-    process.env.CI === 'true' ||
-    // This will be true for invocations such as `npx eslint ...` and `./node_modules/.bin/eslint ...`
-    process.argv[1].endsWith(normalize('node_modules/.bin/eslint'))
-  ) {
+  if (process.env.TSESTREE_SINGLE_RUN === 'true') {
     extra.singleRun = true;
     return;
   }
 
+  // Currently behind a flag while we gather real-world feedback
+  if (options?.allowAutomaticSingleRunInference) {
+    if (
+      // Default to single runs for CI processes. CI=true is set by most CI providers by default.
+      process.env.CI === 'true' ||
+      // This will be true for invocations such as `npx eslint ...` and `./node_modules/.bin/eslint ...`
+      process.argv[1].endsWith(normalize('node_modules/.bin/eslint'))
+    ) {
+      extra.singleRun = true;
+      return;
+    }
+  }
+
   /**
    * We default to assuming that this run could be part of a long-running session (e.g. in an IDE)
    * and watch programs will therefore be required
@@ -465,7 +469,7 @@ function parseWithNodeMapsInternal<T extends TSESTreeOptions = TSESTreeOptions>(
   /**
    * Figure out whether this is a single run or part of a long-running process
    */
-  inferSingleRun();
+  inferSingleRun(options);
 
   /**
    * Create a ts.SourceFile directly, no ts.Program is needed for a simple
@@ -529,27 +533,32 @@ function parseAndGenerateServices<T extends TSESTreeOptions = TSESTreeOptions>(
   /**
    * Figure out whether this is a single run or part of a long-running process
    */
-  inferSingleRun();
+  inferSingleRun(options);
 
   /**
    * If this is a single run in which the user has not provided any existing programs but there
    * are programs which need to be created from the provided "project" option,
-   * create the programs once ahead of time and avoid watch programs
+   * create an Iterable which will lazily create the programs as needed by the iteration logic
    */
   if (extra.singleRun && !extra.programs && extra.projects?.length > 0) {
-    extra.programs = extra.projects.map(configFile => {
-      const existingProgram = existingPrograms.get(configFile);
-      if (existingProgram) {
-        return existingProgram;
-      }
-      log(
-        'Detected single-run/CLI usage, creating Program once ahead of time for project: %s',
-        configFile,
-      );
-      const newProgram = createProgramFromConfigFile(configFile);
-      existingPrograms.set(configFile, newProgram);
-      return newProgram;
-    });
+    extra.programs = {
+      *[Symbol.iterator](): Iterator<ts.Program> {
+        for (const configFile of extra.projects) {
+          const existingProgram = existingPrograms.get(configFile);
+          if (existingProgram) {
+            yield existingProgram;
+          } else {
+            log(
+              'Detected single-run/CLI usage, creating Program once ahead of time for project: %s',
+              configFile,
+            );
+            const newProgram = createProgramFromConfigFile(configFile);
+            existingPrograms.set(configFile, newProgram);
+            yield newProgram;
+          }
+        }
+      },
+    };
   }
 
   /**
diff --git a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
index bfd4a85da13..a89d929d5a3 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
@@ -6,6 +6,9 @@ const mockProgram = {
   getSourceFile(): void {
     return;
   },
+  getTypeChecker(): void {
+    return;
+  },
 };
 
 jest.mock('../../src/ast-converter', () => {
@@ -16,18 +19,45 @@ jest.mock('../../src/ast-converter', () => {
   };
 });
 
+interface MockProgramWithConfigFile {
+  __FROM_CONFIG_FILE__?: string;
+}
+
+jest.mock('../../src/create-program/shared.ts', () => {
+  return {
+    ...jest.requireActual('../../src/create-program/shared.ts'),
+    getAstFromProgram(program: MockProgramWithConfigFile): unknown {
+      if (
+        program.__FROM_CONFIG_FILE__?.endsWith('non-matching-tsconfig.json')
+      ) {
+        return null;
+      }
+      // Remove temporary tracking value for the config added by mock createProgramFromConfigFile() below
+      delete program.__FROM_CONFIG_FILE__;
+      return { ast: {}, program };
+    },
+  };
+});
+
 jest.mock('../../src/create-program/useProvidedPrograms.ts', () => {
   return {
     ...jest.requireActual('../../src/create-program/useProvidedPrograms.ts'),
-    useProvidedPrograms: jest.fn(() => {
-      return {
-        ast: {
-          parseDiagnostics: [],
-        },
-        program: mockProgram,
-      };
-    }),
-    createProgramFromConfigFile: jest.fn(() => mockProgram),
+    createProgramFromConfigFile: jest
+      .fn()
+      .mockImplementation((configFile): MockProgramWithConfigFile => {
+        return {
+          // So we can differentiate our mock return values based on which tsconfig this is
+          __FROM_CONFIG_FILE__: configFile,
+          ...mockProgram,
+        };
+      }),
+  };
+});
+
+jest.mock('../../src/create-program/createWatchProgram', () => {
+  return {
+    ...jest.requireActual('../../src/create-program/createWatchProgram'),
+    getProgramsForProjects: jest.fn(() => [mockProgram]),
   };
 });
 
@@ -41,7 +71,8 @@ const testFiles = glob.sync(`**/*.src.ts`, {
 });
 
 const code = 'const foo = 5;';
-const tsconfigs = ['./tsconfig.json', './badTSConfig/tsconfig.json'];
+// File will not be found in the first tsconfig's Program, but will be in the second
+const tsconfigs = ['./non-matching-tsconfig.json', './tsconfig.json'];
 const options = {
   filePath: testFiles[0],
   tsconfigRootDir: path.join(process.cwd(), FIXTURES_DIR),
@@ -85,7 +116,7 @@ describe('semanticInfo - singleRun', () => {
     process.env.CI = originalEnvCI;
   });
 
-  it('should create all programs for provided "parserOptions.project" once ahead of time when TSESTREE_SINGLE_RUN=true', () => {
+  it('should lazily create the required program out of the provided "parserOptions.project" one time when TSESTREE_SINGLE_RUN=true', () => {
     /**
      * Single run because of explicit environment variable TSESTREE_SINGLE_RUN
      */
@@ -94,7 +125,7 @@ describe('semanticInfo - singleRun', () => {
 
     const resultProgram = parseAndGenerateServices(code, options).services
       .program;
-    expect(resultProgram).toBe(mockProgram);
+    expect(resultProgram).toEqual(mockProgram);
 
     // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
     parseAndGenerateServices(code, options);
@@ -114,7 +145,7 @@ describe('semanticInfo - singleRun', () => {
     process.env.TSESTREE_SINGLE_RUN = originalTSESTreeSingleRun;
   });
 
-  it('should create all programs for provided "parserOptions.project" once ahead of time when singleRun is inferred from CI=true', () => {
+  it('should lazily create the required program out of the provided "parserOptions.project" one time when singleRun is inferred from CI=true', () => {
     /**
      * Single run because of CI=true (we need to make sure we respect the original value
      * so that we won't interfere with our own usage of the variable)
@@ -124,7 +155,7 @@ describe('semanticInfo - singleRun', () => {
 
     const resultProgram = parseAndGenerateServices(code, options).services
       .program;
-    expect(resultProgram).toBe(mockProgram);
+    expect(resultProgram).toEqual(mockProgram);
 
     // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
     parseAndGenerateServices(code, options);
@@ -144,7 +175,7 @@ describe('semanticInfo - singleRun', () => {
     process.env.CI = originalEnvCI;
   });
 
-  it('should create all programs for provided "parserOptions.project" once ahead of time when singleRun is inferred from process.argv', () => {
+  it('should lazily create the required program out of the provided "parserOptions.project" one time when singleRun is inferred from process.argv', () => {
     /**
      * Single run because of process.argv
      */
@@ -153,7 +184,7 @@ describe('semanticInfo - singleRun', () => {
 
     const resultProgram = parseAndGenerateServices(code, options).services
       .program;
-    expect(resultProgram).toBe(mockProgram);
+    expect(resultProgram).toEqual(mockProgram);
 
     // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
     parseAndGenerateServices(code, options);
@@ -172,4 +203,37 @@ describe('semanticInfo - singleRun', () => {
     // Restore process data
     process.argv = originalProcessArgv;
   });
+
+  it('should stop iterating through and lazily creating programs for the given "parserOptions.project" once a matching one has been found', () => {
+    /**
+     * Single run because of explicit environment variable TSESTREE_SINGLE_RUN
+     */
+    const originalTSESTreeSingleRun = process.env.TSESTREE_SINGLE_RUN;
+    process.env.TSESTREE_SINGLE_RUN = 'true';
+
+    const optionsWithReversedTsconfigs = {
+      ...options,
+      //  Now the matching tsconfig comes first
+      project: options.project.reverse(),
+    };
+
+    const resultProgram = parseAndGenerateServices(
+      code,
+      optionsWithReversedTsconfigs,
+    ).services.program;
+    expect(resultProgram).toEqual(mockProgram);
+
+    // Call parseAndGenerateServices() again to ensure caching of Programs is working correctly...
+    parseAndGenerateServices(code, options);
+    // ...by asserting this was only called only once
+    expect(createProgramFromConfigFile).toHaveBeenCalledTimes(1);
+
+    expect(createProgramFromConfigFile).toHaveBeenNthCalledWith(
+      1,
+      resolvedProject(tsconfigs[0]),
+    );
+
+    // Restore process data
+    process.env.TSESTREE_SINGLE_RUN = originalTSESTreeSingleRun;
+  });
 });

From d307cc290ee5b30f323a6f87f132a0f64e54419d Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 21:30:33 +0400
Subject: [PATCH 3/7] fix: misc

---
 packages/typescript-estree/src/index.ts                        | 2 +-
 .../typescript-estree/tests/lib/semanticInfo-singleRun.test.ts | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/typescript-estree/src/index.ts b/packages/typescript-estree/src/index.ts
index d00e8a1d953..b2a0581dc92 100644
--- a/packages/typescript-estree/src/index.ts
+++ b/packages/typescript-estree/src/index.ts
@@ -2,7 +2,7 @@ export * from './parser';
 export { ParserServices, TSESTreeOptions } from './parser-options';
 export { simpleTraverse } from './simple-traverse';
 export * from './ts-estree';
-export { clearWatchCaches } from './create-program/createWatchProgram';
+export { clearWatchCaches as clearCaches } from './create-program/createWatchProgram';
 export { createProgramFromConfigFile as createProgram } from './create-program/useProvidedPrograms';
 
 // re-export for backwards-compat
diff --git a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
index a89d929d5a3..68ad28efa24 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
@@ -71,13 +71,14 @@ const testFiles = glob.sync(`**/*.src.ts`, {
 });
 
 const code = 'const foo = 5;';
-// File will not be found in the first tsconfig's Program, but will be in the second
+// File will not be found in the first Program, but will be in the second
 const tsconfigs = ['./non-matching-tsconfig.json', './tsconfig.json'];
 const options = {
   filePath: testFiles[0],
   tsconfigRootDir: path.join(process.cwd(), FIXTURES_DIR),
   loggerFn: false,
   project: tsconfigs,
+  allowAutomaticSingleRunInference: true,
 } as const;
 
 const resolvedProject = (p: string): string =>

From 1012676d9a2f7c56ef1e03e3efa80d454948bd3b Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 21:36:18 +0400
Subject: [PATCH 4/7] fix: clearCache usage

---
 .../src/eslint-utils/RuleTester.ts                 |  2 +-
 packages/parser/src/index.ts                       |  2 +-
 packages/typescript-estree/tests/lib/parse.test.ts |  2 +-
 .../tests/lib/persistentParse.test.ts              |  3 ++-
 .../tests/lib/semanticInfo.test.ts                 | 14 +++++++-------
 5 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/packages/experimental-utils/src/eslint-utils/RuleTester.ts b/packages/experimental-utils/src/eslint-utils/RuleTester.ts
index 7479684393c..a3210162e98 100644
--- a/packages/experimental-utils/src/eslint-utils/RuleTester.ts
+++ b/packages/experimental-utils/src/eslint-utils/RuleTester.ts
@@ -33,7 +33,7 @@ class RuleTester extends TSESLint.RuleTester {
         // instead of creating a hard dependency, just use a soft require
         // a bit weird, but if they're using this tooling, it'll be installed
         // eslint-disable-next-line @typescript-eslint/no-unsafe-call
-        require(parser).clearWatchCaches();
+        require(parser).clearCaches();
       } catch {
         // ignored
       }
diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts
index 1ae825f7537..c1bb82bf593 100644
--- a/packages/parser/src/index.ts
+++ b/packages/parser/src/index.ts
@@ -1,7 +1,7 @@
 export { parse, parseForESLint, ParserOptions } from './parser';
 export {
   ParserServices,
-  clearWatchCaches,
+  clearCaches,
   createProgram,
 } from '@typescript-eslint/typescript-estree';
 
diff --git a/packages/typescript-estree/tests/lib/parse.test.ts b/packages/typescript-estree/tests/lib/parse.test.ts
index cfd4db39acf..3e312973700 100644
--- a/packages/typescript-estree/tests/lib/parse.test.ts
+++ b/packages/typescript-estree/tests/lib/parse.test.ts
@@ -632,7 +632,7 @@ describe('parseAndGenerateServices', () => {
 
   describe('projectFolderIgnoreList', () => {
     beforeEach(() => {
-      parser.clearWatchCaches();
+      parser.clearCaches();
     });
 
     const PROJECT_DIR = resolve(FIXTURES_DIR, '../projectFolderIgnoreList');
diff --git a/packages/typescript-estree/tests/lib/persistentParse.test.ts b/packages/typescript-estree/tests/lib/persistentParse.test.ts
index 5de5ed7a471..50ab0351c16 100644
--- a/packages/typescript-estree/tests/lib/persistentParse.test.ts
+++ b/packages/typescript-estree/tests/lib/persistentParse.test.ts
@@ -1,7 +1,8 @@
 import fs from 'fs';
 import path from 'path';
 import tmp from 'tmp';
-import { clearWatchCaches, parseAndGenerateServices } from '../../src';
+import { clearWatchCaches } from '../../src/create-program/createWatchProgram';
+import { parseAndGenerateServices } from '../../src/parser';
 
 const CONTENTS = {
   foo: 'console.log("foo")',
diff --git a/packages/typescript-estree/tests/lib/semanticInfo.test.ts b/packages/typescript-estree/tests/lib/semanticInfo.test.ts
index 7c448362b98..d099342c72e 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo.test.ts
@@ -2,19 +2,19 @@ import * as fs from 'fs';
 import glob from 'glob';
 import * as path from 'path';
 import * as ts from 'typescript';
+import { clearWatchCaches } from '../../src/create-program/createWatchProgram';
+import { createProgramFromConfigFile as createProgram } from '../../src/create-program/useProvidedPrograms';
+import {
+  parseAndGenerateServices,
+  ParseAndGenerateServicesResult,
+} from '../../src/parser';
 import { TSESTreeOptions } from '../../src/parser-options';
+import { TSESTree } from '../../src/ts-estree';
 import {
   createSnapshotTestBlock,
   formatSnapshotName,
   parseCodeAndGenerateServices,
 } from '../../tools/test-utils';
-import {
-  clearWatchCaches,
-  createProgram,
-  parseAndGenerateServices,
-  ParseAndGenerateServicesResult,
-} from '../../src';
-import { TSESTree } from '../../src/ts-estree';
 
 const FIXTURES_DIR = './tests/fixtures/semanticInfo';
 const testFiles = glob.sync(`**/*.src.ts`, {

From 0faca588cb7e5ea8777e309d03f9f7037f32d006 Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 21:43:02 +0400
Subject: [PATCH 5/7] test: leverage getCanonicalFileName

---
 .../typescript-estree/tests/lib/semanticInfo-singleRun.test.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
index 68ad28efa24..4d775a04f3f 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
@@ -1,6 +1,7 @@
 import glob from 'glob';
 import * as path from 'path';
 import { clearProgramCache, parseAndGenerateServices } from '../../src';
+import { getCanonicalFileName } from '../../src/create-program/shared';
 
 const mockProgram = {
   getSourceFile(): void {
@@ -82,7 +83,7 @@ const options = {
 } as const;
 
 const resolvedProject = (p: string): string =>
-  path.resolve(path.join(process.cwd(), FIXTURES_DIR), p).toLowerCase();
+  getCanonicalFileName(path.resolve(path.join(process.cwd(), FIXTURES_DIR), p));
 
 describe('semanticInfo - singleRun', () => {
   beforeEach(() => {

From 4137b8ac4b55da11e7c4f8fd18a1eafee87503fc Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 21:54:17 +0400
Subject: [PATCH 6/7] fix: ignore real CI=true for particular spec

---
 .../tests/lib/semanticInfo-singleRun.test.ts             | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
index 4d775a04f3f..b8e778e0fb0 100644
--- a/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
+++ b/packages/typescript-estree/tests/lib/semanticInfo-singleRun.test.ts
@@ -94,12 +94,19 @@ describe('semanticInfo - singleRun', () => {
   });
 
   it('should not create any programs ahead of time by default when there is no way to infer singleRun=true', () => {
+    // For when these tests themselves are running in CI, we need to ignore that for this particular spec
+    const originalEnvCI = process.env.CI;
+    process.env.CI = 'false';
+
     /**
-     * Nothing to indicate it is a single run, so createProgramFromConfigFile should
+     * At this point there is nothing to indicate it is a single run, so createProgramFromConfigFile should
      * never be called
      */
     parseAndGenerateServices(code, options);
     expect(createProgramFromConfigFile).not.toHaveBeenCalled();
+
+    // Restore process data
+    process.env.CI = originalEnvCI;
   });
 
   it('should not create any programs ahead of time when when TSESTREE_SINGLE_RUN=false, even if other inferrence criteria apply', () => {

From 9663f12023936f5a519935177e797fdeeb768866 Mon Sep 17 00:00:00 2001
From: James Henry <james@henry.sc>
Date: Fri, 11 Jun 2021 23:22:10 +0400
Subject: [PATCH 7/7] Update
 packages/typescript-estree/src/create-program/useProvidedPrograms.ts

Co-authored-by: Brad Zacher <brad.zacher@gmail.com>
---
 .../typescript-estree/src/create-program/useProvidedPrograms.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/typescript-estree/src/create-program/useProvidedPrograms.ts b/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
index dd12f17461a..23b56497dc6 100644
--- a/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
+++ b/packages/typescript-estree/src/create-program/useProvidedPrograms.ts
@@ -12,7 +12,7 @@ import {
 const log = debug('typescript-eslint:typescript-estree:useProvidedProgram');
 
 function useProvidedPrograms(
-  programInstances: ts.Program[] | Iterable<ts.Program>,
+  programInstances: Iterable<ts.Program>,
   extra: Extra,
 ): ASTAndProgram | undefined {
   log(