feat: refactor to split AST specification out as its own module

Fixes #2726
Fixes #2912

This PR is the basis for a big cleanup and reorganisation of the AST.
This first step takes the file `types/src/ts-estree.ts` and splits it up in its entirety.
This file was a monolith at 1700 lines - meaning it was a pain to organise and manage, and there was no way to isolate/restrict certain things (aside from adding comments).

This PR should ultimately be a no-op - there should be no breaking changes here.
I did fix up a number of types which I found when organising files into their folders.

Whilst this PR ultimately creates more LOC, the isolation enables a few things:
- By splitting the AST into its own module, it's isolated so easier to manage / govern
- By splitting each AST node into its own folder we can cleanly document and link to individual node specs
- By grouping nodes decls by folder, it's easier to inspect the types to validate unions are correct.
    - I found a number of invalid nodes in unions in this PR which have been fixed.
- In a future PR we can:
    - Add lint rule(s) to validate unions are correct (eg ensure all `Expression` types are included in the `Expression` union).
    - Easily add documentation about the node without cluttering things up
    - Colocate fixtures/snapshots with the node specs to document the cases that we expect a node to show up
    - Colocate the conversion logic here so that it's easier to validate that the spec and the conversion logic are in sync
        - This will make it much easier to implement and maintain #1852

.eslintignore

@@ -7,3 +7,6 @@ coverage
 __snapshots__
 
 packages/eslint-plugin-tslint/tests
+
+# Files copied as part of the build
+packages/types/src/ast-spec.ts

.eslintrc.js

@@ -7,6 +7,7 @@ module.exports = {
     'import',
     'eslint-comments',
     '@typescript-eslint/internal',
+    'simple-import-sort',
   ],
   env: {
     es6: true,
@@ -261,5 +262,27 @@ module.exports = {
         '@typescript-eslint/internal/prefer-ast-types-enum': 'off',
       },
     },
+    // ast spec specific standardization
+    {
+      files: ['packages/ast-spec/src/**/*.ts'],
+      rules: {
+        '@typescript-eslint/consistent-type-imports': [
+          'error',
+          { prefer: 'type-imports', disallowTypeAnnotations: true },
+        ],
+        '@typescript-eslint/no-unused-vars': 'error',
+        '@typescript-eslint/sort-type-union-intersection-members': 'error',
+        'import/first': 'error',
+        'import/newline-after-import': 'error',
+        'import/no-duplicates': 'error',
+        'simple-import-sort/imports': 'error',
+      },
+    },
+    {
+      files: ['rollup.config.ts'],
+      rules: {
+        'import/no-default-export': 'off',
+      },
+    },
   ],
 };

.gitignore

@@ -67,3 +67,7 @@ dist
 _ts3.4
 *.tsbuildinfo
 .watchmanconfig
+.rollup.cache
+
+# Files copied as part of the build
+packages/types/src/ast-spec.ts

.vscode/settings.json

@@ -13,8 +13,8 @@
 
   // typescript auto-format settings
   "typescript.tsdk": "node_modules/typescript/lib",
-  "javascript.preferences.importModuleSpecifier": "auto",
-  "typescript.preferences.importModuleSpecifier": "auto",
+  "javascript.preferences.importModuleSpecifier": "project-relative",
+  "typescript.preferences.importModuleSpecifier": "project-relative",
   "javascript.preferences.quoteStyle": "single",
   "typescript.preferences.quoteStyle": "single",
   "editor.defaultFormatter": "esbenp.prettier-vscode",

package.json

@@ -17,7 +17,7 @@
     "url": "https://github.com/typescript-eslint/typescript-eslint/issues"
   },
   "scripts": {
-    "build": "lerna run build",
+    "build": "lerna run build --ignore ast-spec",
     "check:clean-workspace-after-install": "git diff --quiet --exit-code",
     "check:configs": "lerna run check:configs",
     "check:docs": "lerna run check:docs",
@@ -98,6 +98,7 @@
     "eslint-plugin-eslint-plugin": "^3.0.0",
     "eslint-plugin-import": "^2.22.0",
     "eslint-plugin-jest": "^24.1.3",
+    "eslint-plugin-simple-import-sort": "^7.0.0",
     "glob": "^7.1.6",
     "husky": "^5.0.9",
     "jest": "^26.6.3",

packages/ast-spec/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 TypeScript ESLint and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/ast-spec/README.md

@@ -0,0 +1,24 @@
+<h1 align="center">TypeScript-ESTree AST Specification</h1>
+
+<p align="center">
+    <img src="https://github.com/typescript-eslint/typescript-eslint/workflows/CI/badge.svg" alt="CI" />
+    <a href="https://www.npmjs.com/package/@typescript-eslint/ast-spec"><img src="https://img.shields.io/npm/v/@typescript-eslint/ast-spec.svg?style=flat-square" alt="NPM Version" /></a>
+    <a href="https://www.npmjs.com/package/@typescript-eslint/ast-spec"><img src="https://img.shields.io/npm/dm/@typescript-eslint/ast-spec.svg?style=flat-square" alt="NPM Downloads" /></a>
+</p>
+
+This is the complete specification for the TypeScript-ESTree AST.
+
+It includes:
+
+- Node definitions as TypeScript types (the specification)
+- Logic for converting from the TypeScript AST to the TypeScript-ESTree AST.
+- Tests/Fixtures/Examples for each Node
+
+**You probably don't want to use it directly.**
+
+If you're building an ESLint plugin, consider using [`@typescript-eslint/experimental-utils`](../experimental-utils).
+If you're parsing TypeScript code, consider using [`@typescript-eslint/typescript-estree`](../typescript-estree).
+
+## Contributing
+
+[See the contributing guide here](../../CONTRIBUTING.md)

packages/ast-spec/api-extractor.json

@@ -0,0 +1,31 @@
+{
+  "mainEntryPointFilePath": "<projectFolder>/dist/index.d.ts",
+  "apiReport": {
+    "enabled": false
+  },
+  "docModel": {
+    "enabled": false
+  },
+  "dtsRollup": {
+    "enabled": true,
+    "untrimmedFilePath": "<projectFolder>/dist/ast-spec.ts"
+  },
+  "tsdocMetadata": {
+    "enabled": false
+  },
+  "messages": {
+    "extractorMessageReporting": {
+      "default": {
+        "logLevel": "none"
+      },
+      "ae-forgotten-export": {
+        "logLevel": "none"
+      }
+    },
+    "tsdocMessageReporting": {
+      "default": {
+        "logLevel": "none"
+      }
+    }
+  }
+}

packages/ast-spec/jest.config.js

@@ -0,0 +1,20 @@
+'use strict';
+
+// @ts-check
+/** @type {import('@jest/types').Config.InitialOptions} */
+module.exports = {
+  globals: {
+    'ts-jest': {
+      isolatedModules: true,
+    },
+  },
+  testEnvironment: 'node',
+  transform: {
+    ['^.+\\.tsx?$']: 'ts-jest',
+  },
+  testRegex: ['./tests/.+\\.test\\.ts$'],
+  collectCoverage: false,
+  collectCoverageFrom: ['src/**/*.{js,jsx,ts,tsx}'],
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  coverageReporters: ['text-summary', 'lcov'],
+};

packages/ast-spec/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@typescript-eslint/ast-spec",
+  "version": "4.20.0",
+  "description": "TypeScript-ESTree AST spec",
+  "private": true,
+  "keywords": [
+    "eslint",
+    "typescript",
+    "estree"
+  ],
+  "engines": {
+    "node": "^8.10.0 || ^10.13.0 || >=11.10.1"
+  },
+  "files": [
+    "dist",
+    "package.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/typescript-eslint/typescript-eslint.git",
+    "directory": "packages/ast-spec"
+  },
+  "bugs": {
+    "url": "https://github.com/typescript-eslint/typescript-eslint/issues"
+  },
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc -b tsconfig.build.json && api-extractor run --local",
+    "clean": "tsc -b tsconfig.build.json --clean",
+    "postclean": "rimraf dist && rimraf _ts3.4 && rimraf .rollup.cache && rimraf coverage",
+    "format": "prettier --write \"./**/*.{ts,js,json,md}\" --ignore-path ../../.prettierignore",
+    "lint": "eslint . --ext .js,.ts --ignore-path='../../.eslintignore'",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "funding": {
+    "type": "opencollective",
+    "url": "https://opencollective.com/typescript-eslint"
+  },
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.13.2"
+  }
+}

packages/types/src/ast-node-types.ts → packages/ast-spec/src/ast-node-types.ts

@@ -1,4 +1,4 @@
-enum AST_NODE_TYPES {
+export enum AST_NODE_TYPES {
   ArrayExpression = 'ArrayExpression',
   ArrayPattern = 'ArrayPattern',
   ArrowFunctionExpression = 'ArrowFunctionExpression',
@@ -164,23 +164,3 @@ enum AST_NODE_TYPES {
   TSUnknownKeyword = 'TSUnknownKeyword',
   TSVoidKeyword = 'TSVoidKeyword',
 }
-
-export { AST_NODE_TYPES };
-
-// Below is a special type-only test which ensures that we don't accidentally leave unused keys in this enum
-// eslint-disable-next-line import/first -- purposely down here to colocate it with this hack of a test
-import type { Node } from './ts-estree';
-
-type GetKeys<T extends AST_NODE_TYPES> = keyof Extract<Node, { type: T }>;
-
-type AllKeys = {
-  readonly [T in AST_NODE_TYPES]: GetKeys<T>;
-};
-
-type TakesString<T extends Record<string, string>> = T;
-
-// @ts-expect-error: purposely unused
-type _Test =
-  // forcing the test onto a new line so it isn't covered by the expect error
-  // If there are any enum members that don't have a corresponding TSESTree.Node, then this line will error with "Type 'string | number | symbol' is not assignable to type 'string'."
-  void | TakesString<AllKeys>;

packages/types/src/ast-token-types.ts → packages/ast-spec/src/ast-token-types.ts

@@ -1,4 +1,4 @@
-enum AST_TOKEN_TYPES {
+export enum AST_TOKEN_TYPES {
   Boolean = 'Boolean',
   Identifier = 'Identifier',
   JSXIdentifier = 'JSXIdentifier',
@@ -15,5 +15,3 @@ enum AST_TOKEN_TYPES {
   Block = 'Block',
   Line = 'Line',
 }
-
-export { AST_TOKEN_TYPES };

packages/ast-spec/src/base/Accessibility.ts

@@ -0,0 +1 @@
+export type Accessibility = 'private' | 'protected' | 'public';

packages/ast-spec/src/base/BaseNode.ts

@@ -0,0 +1,22 @@
+// import type { Node } from '../unions/Node';
+import type { Range } from './Range';
+import type { SourceLocation } from './SourceLocation';
+
+export interface BaseNode {
+  /**
+   * The source location information of the node.
+   * @see {SourceLocation}
+   */
+  loc: SourceLocation;
+  /**
+   * @see {Range}
+   */
+  range: Range;
+  /**
+   * The parent node of the current node
+   */
+  // parent?: Node;
+
+  // every node *will* have a type, but let the nodes define their own exact string
+  // type: string;
+}

packages/ast-spec/src/base/BaseToken.ts

@@ -0,0 +1,8 @@
+import type { BaseNode } from './BaseNode';
+
+/*
+ * Token and Comment are pseudo-nodes to represent pieces of source code
+ */
+export interface BaseToken extends BaseNode {
+  value: string;
+}

packages/ast-spec/src/base/BinaryExpressionBase.ts

@@ -0,0 +1,8 @@
+import type { Expression } from '../unions/Expression';
+import type { BaseNode } from './BaseNode';
+
+export interface BinaryExpressionBase extends BaseNode {
+  operator: string;
+  left: Expression;
+  right: Expression;
+}

packages/ast-spec/src/base/ClassDeclarationBase.ts

@@ -0,0 +1,20 @@
+import type { Identifier } from '../expression/Identifier/spec';
+import type { ClassBody } from '../special/ClassBody/spec';
+import type { Decorator } from '../special/Decorator/spec';
+import type { TSClassImplements } from '../special/TSClassImplements/spec';
+import type { TSTypeParameterDeclaration } from '../special/TSTypeParameterDeclaration/spec';
+import type { TSTypeParameterInstantiation } from '../special/TSTypeParameterInstantiation/spec';
+import type { LeftHandSideExpression } from '../unions/LeftHandSideExpression';
+import type { BaseNode } from './BaseNode';
+
+export interface ClassDeclarationBase extends BaseNode {
+  typeParameters?: TSTypeParameterDeclaration;
+  superTypeParameters?: TSTypeParameterInstantiation;
+  id: Identifier | null;
+  body: ClassBody;
+  superClass: LeftHandSideExpression | null;
+  implements?: TSClassImplements[];
+  abstract?: boolean;
+  declare?: boolean;
+  decorators?: Decorator[];
+}

packages/ast-spec/src/base/ClassPropertyBase.ts

@@ -0,0 +1,34 @@
+import type { Decorator } from '../special/Decorator/spec';
+import type { TSTypeAnnotation } from '../special/TSTypeAnnotation/spec';
+import type { Expression } from '../unions/Expression';
+import type {
+  PropertyName,
+  PropertyNameComputed,
+  PropertyNameNonComputed,
+} from '../unions/PropertyName';
+import type { Accessibility } from './Accessibility';
+import type { BaseNode } from './BaseNode';
+
+interface ClassPropertyBase extends BaseNode {
+  key: PropertyName;
+  value: Expression | null;
+  computed: boolean;
+  static: boolean;
+  declare: boolean;
+  readonly?: boolean;
+  decorators?: Decorator[];
+  accessibility?: Accessibility;
+  optional?: boolean;
+  definite?: boolean;
+  typeAnnotation?: TSTypeAnnotation;
+}
+
+export interface ClassPropertyComputedNameBase extends ClassPropertyBase {
+  key: PropertyNameComputed;
+  computed: true;
+}
+
+export interface ClassPropertyNonComputedNameBase extends ClassPropertyBase {
+  key: PropertyNameNonComputed;
+  computed: false;
+}

packages/ast-spec/src/base/FunctionDeclarationBase.ts

@@ -0,0 +1,18 @@
+import type { Identifier } from '../expression/Identifier/spec';
+import type { TSTypeAnnotation } from '../special/TSTypeAnnotation/spec';
+import type { TSTypeParameterDeclaration } from '../special/TSTypeParameterDeclaration/spec';
+import type { BlockStatement } from '../statement/BlockStatement/spec';
+import type { Parameter } from '../unions/Parameter';
+import type { BaseNode } from './BaseNode';
+
+export interface FunctionDeclarationBase extends BaseNode {
+  id: Identifier | null;
+  generator: boolean;
+  expression: boolean;
+  async: boolean;
+  params: Parameter[];
+  body?: BlockStatement | null;
+  returnType?: TSTypeAnnotation;
+  typeParameters?: TSTypeParameterDeclaration;
+  declare?: boolean;
+}

packages/ast-spec/src/base/LineAndColumnData.ts

@@ -0,0 +1,10 @@
+export interface LineAndColumnData {
+  /**
+   * Line number (1-indexed)
+   */
+  line: number;
+  /**
+   * Column number on the line (0-indexed)
+   */
+  column: number;
+}