Support .mts, .cts, nodenext, and node16 (#1694)

* feat: support .mts .cts

* stuff

* WIP with TODOs

* WIP

* lint-fix

* WIP

* WIP

* WIP

* fixes

* flatten resolver tests by splitting into more  functions

* address extension-handling code

* lint-fix

* Fix; update resolver tests to test more things

* fix

* fix

* test against TS rc on nodes 14, 16, 18 (good sanity-checking with these new nodenext features)

* Teach ts.transpileModule to handle NodeNext correctly

* tweak tests

* fix test typos; update pluggable dep tests; update ignore() tests for new file extensions

* fix tests

* fix

* fix build against stable ts, no need for 4.7.0 just yet

* Gate nodenext/node16 tests behind a TS version check; fix TSCommon types

* Fix nyc require.extensions issues

* lint-fix

* tricky types

* skip nodenext tests on older TS

* fix tests

* fix bug and tests

* fix windows

* turn off another test on ancient TS versions

* fix windows

* fix allowing `moduleTypeOverrides` to override cts/cjs

* Update moduleTypes docs

* add mts and cts awareness to internal/external classifier; add .d.cts/.d.mts extensions to tests

* cleanup misc markdown files

* more markdown cleanup

Co-authored-by: bluelovers <codelovers@users.sourceforge.net>

.github/workflows/continuous-integration.yml

@@ -51,7 +51,7 @@ jobs:
       matrix:
         os: [ubuntu, windows]
         # Don't forget to add all new flavors to this list!
-        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
         include:
           # Node 12.15
           - flavor: 1
@@ -95,41 +95,64 @@ jobs:
             nodeFlag: 14
             typescript: next
             typescriptFlag: next
+          - flavor: 8
+            node: 14
+            nodeFlag: 14
+            typescript: rc
+            typescriptFlag: rc
           # Node 16
           # Node 16.11.1
           # Earliest version that supports old ESM Loader Hooks API: https://github.com/TypeStrong/ts-node/pull/1522
-          - flavor: 8
+          - flavor: 9
             node: 16.11.1
             nodeFlag: 16_11_1
             typescript: latest
             typescriptFlag: latest
-          - flavor: 9
+          - flavor: 10
             node: 16
             nodeFlag: 16
             typescript: latest
             typescriptFlag: latest
             downgradeNpm: true
-          - flavor: 10
+          - flavor: 11
             node: 16
             nodeFlag: 16
             typescript: 2.7
             typescriptFlag: 2_7
             downgradeNpm: true
-          - flavor: 11
+          - flavor: 12
             node: 16
             nodeFlag: 16
             typescript: next
             typescriptFlag: next
             downgradeNpm: true
+          - flavor: 13
+            node: 16
+            nodeFlag: 16
+            typescript: rc
+            typescriptFlag: rc
+            downgradeNpm: true
           # Node 18
-          - flavor: 12
+          - flavor: 14
             node: 18
             nodeFlag: 18
             typescript: latest
             typescriptFlag: latest
             downgradeNpm: true
+          - flavor: 15
+            node: 18
+            nodeFlag: 18
+            typescript: next
+            typescriptFlag: next
+            downgradeNpm: true
+          - flavor: 16
+            node: 18
+            nodeFlag: 18
+            typescript: rc
+            typescriptFlag: rc
+            downgradeNpm: true
           # Node nightly
-          - flavor: 13
+          - flavor: 17
             node: nightly
             nodeFlag: nightly
             typescript: latest

.vscode/launch.json

@@ -10,29 +10,33 @@
       "outputCapture": "std",
       "skipFiles": [
         "<node_internals>/**/*.js"
-      ],
+      ]
     },
     {
-      "name": "Debug resolver tests (example)",
+      "name": "Debug resolver test",
       "type": "pwa-node",
       "request": "launch",
-      "cwd": "${workspaceFolder}/tests/tmp/resolver-0015-preferSrc-typeModule-allowJs-experimentalSpecifierResolutionNode",
+      "cwd": "${workspaceFolder}/tests/tmp/resolver-0029-preferSrc-typeModule-allowJs-skipIgnore-experimentalSpecifierResolutionNode",
       "runtimeArgs": [
-        "--loader", "../../../esm.mjs"
+        "--loader",
+        "../../../esm.mjs"
       ],
-      "program": "./src/entrypoint-0054-src-to-src.mjs"
+      "program": "./src/entrypoint-0000-src-to-src.cjs"
     },
     {
       "name": "Debug Example: running a test fixture against local ts-node/esm loader",
       "type": "pwa-node",
       "request": "launch",
       "cwd": "${workspaceFolder}/tests/esm",
-      "runtimeArgs": ["--loader", "../../ts-node/esm"],
+      "runtimeArgs": [
+        "--loader",
+        "../../ts-node/esm"
+      ],
       "program": "throw error.ts",
       "outputCapture": "std",
       "skipFiles": [
         "<node_internals>/**/*.js"
-      ],
+      ]
     }
   ]
-}
+}

development-docs/README.md

@@ -0,0 +1,10 @@
+This directory contains a variety of documents:
+
+- notes
+- old to-do lists
+- design ideas from when I implemented various features
+- templates for drafting release notes
+- etc
+
+It is useful to me to keep these notes.  If you find their presence
+confusing, you can safely ignore this directory.

development-docs/nodenextNode16.md

@@ -0,0 +1,53 @@
+# Adding support for NodeNext, Node16, `.cts`, `.mts`, `.cjs`, `.mjs`
+
+*This feature has already been implemented.  Here are my notes from when
+I was doing the work*
+
+## TODOs
+
+Implement node module type classifier:
+- if NodeNext or Node12: ask classifier for CJS or ESM determination
+Add `ForceNodeNextCJSEmit`
+
+Does our code check for .d.ts extensions anywhere?
+- if so, teach it about .d.cts and .d.mts
+
+For nodenext and node12, support supplemental "flavor" information:
+-
+
+Think about splitting out index.ts further:
+- register.ts - hooking stuff
+- types.ts
+- env.ts - env vars and global registration types (process.symbol)
+- service.ts
+
+# TESTS
+
+Matrix:
+
+- package.json type absent, commonjs, and module
+- import and require
+- from cjs and esm
+- .cts, .cjs
+- .mts, .mjs
+- typechecking, transpileOnly, and swc
+- dynamic import
+- import = require
+- static import
+- allowJs on and off
+
+Notes about specific matrix entries:
+- require mjs, mts from cjs throws error
+
+Rethink:
+`getOutput`: null in transpile-only mode.  Also may return emitskipped
+`getOutputTranspileOnly`: configured module option
+`getOutputForceCommonJS`: `commonjs` module option
+`getOutputForceNodeCommonJS`: `nodenext` cjs module option
+`getOutputForceESM`: `esnext` module option
+
+Add second layer of classification to classifier:
+if classifier returns `auto` (no `moduleType` override)
+- if `getOutput` emits, done
+- else call `nodeModuleTypeClassifier`
+  - delegate to appropriate `getOutput` based on its response

notes2.md → development-docs/rootDirOutDirMapping.md

@@ -1,3 +1,5 @@
+## Musings about resolving between rootDir and outDir
+
 When /dist and /src are understood to be overlaid because of src -> dist compiling
 /dist/
 /src/

NOTES.md → development-docs/yarnPnpInterop.md

@@ -1,6 +1,4 @@
-*Delete this file before merging this PR*
-
-## PnP interop
+## Yarn PnP interop
 
 Asked about it here:
 https://discord.com/channels/226791405589233664/654372321225605128/957301175609344070

dist-raw/node-internal-modules-cjs-loader.js

@@ -5,7 +5,10 @@
 'use strict';
 
 const {
+  ArrayIsArray,
+  ArrayPrototypeIncludes,
   ArrayPrototypeJoin,
+  ArrayPrototypePush,
   JSONParse,
   ObjectKeys,
   RegExpPrototypeTest,
@@ -46,6 +49,8 @@ const {
 
 const Module = require('module');
 
+const isWindows = process.platform === 'win32';
+
 let statCache = null;
 
 function stat(filename) {
@@ -133,12 +138,13 @@ function readPackageScope(checkPath) {
 /**
  * @param {{
  *   nodeEsmResolver: ReturnType<typeof import('./node-internal-modules-esm-resolve').createResolve>,
- *   compiledExtensions: string[],
+ *   extensions: import('../src/file-extensions').Extensions,
  *   preferTsExts
  * }} opts
  */
 function createCjsLoader(opts) {
-  const {nodeEsmResolver, compiledExtensions, preferTsExts} = opts;
+const {nodeEsmResolver, preferTsExts} = opts;
+const {replacementsForCjs, replacementsForJs, replacementsForMjs} = opts.extensions;
 const {
   encodedSepRegEx,
   packageExportsResolve,
@@ -209,47 +215,42 @@ function toRealPath(requestPath) {
   });
 }
 
-/**
- * TS's resolver can resolve foo.js to foo.ts, by replacing .js extension with several source extensions.
- * IMPORTANT: preserve ordering according to preferTsExts; this affects resolution behavior!
- */
-const extensions = Array.from(new Set([
-  ...(preferTsExts ? compiledExtensions : []),
-  '.js', '.json', '.node', '.mjs', '.cjs',
-  ...compiledExtensions
-]));
-const replacementExtensions = {
-  '.js': extensions.filter(ext => ['.js', '.jsx', '.ts', '.tsx'].includes(ext)),
-  '.cjs': extensions.filter(ext => ['.cjs', '.cts'].includes(ext)),
-  '.mjs': extensions.filter(ext => ['.mjs', '.mts'].includes(ext)),
-};
-
-const replacableExtensionRe = /(\.(?:js|cjs|mjs))$/;
-
 function statReplacementExtensions(p) {
-  const match = p.match(replacableExtensionRe);
-  if (match) {
-    const replacementExts = replacementExtensions[match[1]];
-    const pathnameWithoutExtension = p.slice(0, -match[1].length);
-    for (let i = 0; i < replacementExts.length; i++) {
-      const filename = pathnameWithoutExtension + replacementExts[i];
-      const rc = stat(filename);
-      if (rc === 0) {
-        return [rc, filename];
+  const lastDotIndex = p.lastIndexOf('.');
+  if(lastDotIndex >= 0) {
+    const ext = p.slice(lastDotIndex);
+    if (ext === '.js' || ext === '.cjs' || ext === '.mjs') {
+      const pathnameWithoutExtension = p.slice(0, lastDotIndex);
+      const replacementExts =
+        ext === '.js' ? replacementsForJs
+        : ext === '.mjs' ? replacementsForMjs
+        : replacementsForCjs;
+      for (let i = 0; i < replacementExts.length; i++) {
+        const filename = pathnameWithoutExtension + replacementExts[i];
+        const rc = stat(filename);
+        if (rc === 0) {
+          return [rc, filename];
+        }
       }
     }
   }
   return [stat(p), p];
 }
 function tryReplacementExtensions(p, isMain) {
-  const match = p.match(replacableExtensionRe);
-  if (match) {
-    const replacementExts = replacementExtensions[match[1]];
-    const pathnameWithoutExtension = p.slice(0, -match[1].length);
-    for (let i = 0; i < replacementExts.length; i++) {
-      const filename = tryFile(pathnameWithoutExtension + replacementExts[i], isMain);
-      if (filename) {
-        return filename;
+  const lastDotIndex = p.lastIndexOf('.');
+  if(lastDotIndex >= 0) {
+    const ext = p.slice(lastDotIndex);
+    if (ext === '.js' || ext === '.cjs' || ext === '.mjs') {
+      const pathnameWithoutExtension = p.slice(0, lastDotIndex);
+      const replacementExts =
+        ext === '.js' ? replacementsForJs
+        : ext === '.mjs' ? replacementsForMjs
+        : replacementsForCjs;
+      for (let i = 0; i < replacementExts.length; i++) {
+        const filename = tryFile(pathnameWithoutExtension + replacementExts[i], isMain);
+        if (filename) {
+          return filename;
+        }
       }
     }
   }
@@ -564,11 +565,18 @@ function assertScriptCanLoadAsCJSImpl(service, module, filename) {
   const pkg = readPackageScope(filename);
 
   // ts-node modification: allow our configuration to override
-  const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename));
+  const tsNodeClassification = service.moduleTypeClassifier.classifyModuleByModuleTypeOverrides(normalizeSlashes(filename));
   if(tsNodeClassification.moduleType === 'cjs') return;
 
+  // ignore package.json when file extension is ESM-only or CJS-only
+  // [MUST_UPDATE_FOR_NEW_FILE_EXTENSIONS]
+  const lastDotIndex = filename.lastIndexOf('.');
+  const ext = lastDotIndex >= 0 ? filename.slice(lastDotIndex) : '';
+
+  if((ext === '.cts' || ext === '.cjs') && tsNodeClassification.moduleType === 'auto') return;
+
   // Function require shouldn't be used in ES modules.
-  if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) {
+  if (ext === '.mts' || ext === '.mjs' || tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) {
     const parentPath = module.parent && module.parent.filename;
     const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null;
     throw createErrRequireEsm(filename, parentPath, packageJsonPath);
@@ -578,5 +586,6 @@ function assertScriptCanLoadAsCJSImpl(service, module, filename) {
 
 module.exports = {
   createCjsLoader,
-  assertScriptCanLoadAsCJSImpl
+  assertScriptCanLoadAsCJSImpl,
+  readPackageScope
 };