Implement ERR_REQUIRE_ESM when a file is require()d which should be…

… loaded as ESM (#1031)

* revert addition of package.json "exports" because it is a breaking change and requires more thought

* Implement ERR_REQUIRE_ESM error when attempting to require() an ESM file

* fix

* test on node 12.15 and 12.16 due to weird ESM handling

* Switch package.json "type" feature detection to a simple version number check

* Remove dead code

* Only throw ERR_REQUIRE_ESM if experimental ESM loader is enabled, since it is a breaking change

* Fix linter failures

* Add tests

* fix test?

* Fix tests

* add comments to explain copy-pasted cjs loader code from node core

* fix

* tweak comment

.github/workflows/continuous-integration.yml

@@ -39,44 +39,49 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu, windows]
-        flavor: [1, 2, 3, 4, 5, 6, 7, 8]
+        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9]
         include:
           - flavor: 1
             node: 10
             nodeFlag: 10
             typescript: latest
             typescriptFlag: latest
           - flavor: 2
-            node: 12
-            nodeFlag: 12
+            node: 12.15
+            nodeFlag: 12_15
             typescript: latest
             typescriptFlag: latest
           - flavor: 3
+            node: 12.16
+            nodeFlag: 12_16
+            typescript: latest
+            typescriptFlag: latest
+          - flavor: 4
             node: 13
             nodeFlag: 13
             typescript: latest
             typescriptFlag: latest
-          - flavor: 4
+          - flavor: 5
             node: 13
             nodeFlag: 13
             typescript: 2.7
             typescriptFlag: 2_7
-          - flavor: 5
+          - flavor: 6
             node: 13
             nodeFlag: 13
             typescript: next
             typescriptFlag: next
-          - flavor: 6
+          - flavor: 7
             node: 14
             nodeFlag: 14
             typescript: latest
             typescriptFlag: latest
-          - flavor: 7
+          - flavor: 8
             node: 14
             nodeFlag: 14
             typescript: 2.7
             typescriptFlag: 2_7
-          - flavor: 8
+          - flavor: 9
             node: 14
             nodeFlag: 14
             typescript: next

dist-raw/README.md

@@ -0,0 +1,13 @@
+The `dist-raw` directory contains JS sources that are distributed verbatim, not compiled nor typechecked via TS.
+
+To implement ESM support, we unfortunately must duplicate some of node's built-in functionality that is not
+exposed via an API.  We have copy-pasted the necessary code from https://github.com/nodejs/node/tree/master/lib
+then modified it to suite our needs.
+
+Formatting may be intentionally bad to keep the diff as small as possible, to make it easier to merge
+upstream changes and understand our modifications.  For example, when we need to wrap node's source code
+in a factory function, we will not indent the function body, to avoid whitespace changes in the diff.
+
+One obvious problem with this approach: the code has been pulled from one version of node, whereas users of ts-node
+run multiple versions of node.
+Users running node 12 may see that ts-node behaves like node 14, for example.

dist-raw/node-cjs-loader-utils.js

@@ -0,0 +1,136 @@
+// Copied from several files in node's source code.
+// https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js
+// Each function and variable below must have a comment linking to the source in node's github repo.
+
+const path = require('path');
+const fs = require('fs');
+
+module.exports.assertScriptCanLoadAsCJSImpl = assertScriptCanLoadAsCJSImpl;
+
+// copied from Module._extensions['.js']
+// https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L1211-L1217
+function assertScriptCanLoadAsCJSImpl(filename) {
+  const pkg = readPackageScope(filename);
+  // Function require shouldn't be used in ES modules.
+  if (pkg && pkg.data && pkg.data.type === 'module') {
+    const parentPath = module.parent && module.parent.filename;
+    const packageJsonPath = path.resolve(pkg.path, 'package.json');
+    throw createErrRequireEsm(filename, parentPath, packageJsonPath);
+  }
+}
+
+// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301
+function readPackageScope(checkPath) {
+  const rootSeparatorIndex = checkPath.indexOf(path.sep);
+  let separatorIndex;
+  while (
+    (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex
+  ) {
+    checkPath = checkPath.slice(0, separatorIndex);
+    if (checkPath.endsWith(path.sep + 'node_modules'))
+      return false;
+    const pjson = readPackage(checkPath);
+    if (pjson) return {
+      path: checkPath,
+      data: pjson
+    };
+  }
+  return false;
+}
+
+// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249
+const packageJsonCache = new Map();
+
+// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L251-L283
+function readPackage(requestPath) {
+  const jsonPath = path.resolve(requestPath, 'package.json');
+
+  const existing = packageJsonCache.get(jsonPath);
+  if (existing !== undefined) return existing;
+
+  const json = internalModuleReadJSON(path.toNamespacedPath(jsonPath));
+  if (json === undefined) {
+    packageJsonCache.set(jsonPath, false);
+    return false;
+  }
+
+  // TODO Related to `--experimental-policy`?  Disabling for now
+  // if (manifest) {
+  //   const jsonURL = pathToFileURL(jsonPath);
+  //   manifest.assertIntegrity(jsonURL, json);
+  // }
+
+  try {
+    const parsed = JSON.parse(json);
+    const filtered = {
+      name: parsed.name,
+      main: parsed.main,
+      exports: parsed.exports,
+      type: parsed.type
+    };
+    packageJsonCache.set(jsonPath, filtered);
+    return filtered;
+  } catch (e) {
+    e.path = jsonPath;
+    e.message = 'Error parsing ' + jsonPath + ': ' + e.message;
+    throw e;
+  }
+}
+
+// In node's core, this is implemented in C
+// https://github.com/nodejs/node/blob/e9f293750760d59243020d0376edf242c9a26b67/src/node_file.cc#L845-L939
+function internalModuleReadJSON(path) {
+  try {
+    return fs.readFileSync(path, 'utf8')
+  } catch (e) {
+    if (e.code === 'ENOENT') return undefined
+    throw e
+  }
+}
+
+// Native ERR_REQUIRE_ESM Error is declared here:
+//   https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313
+// Error class factory is implemented here:
+//   function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341
+//   function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278
+// The code below should create an error that matches the native error as closely as possible.
+// Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error.
+function createErrRequireEsm(filename, parentPath, packageJsonPath) {
+  const code = 'ERR_REQUIRE_ESM'
+  const err = new Error(getMessage(filename, parentPath, packageJsonPath))
+  // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field.
+  // This trick is copied from node's source.
+  err.name = `Error [${ code }]`
+  err.stack
+  Object.defineProperty(err, 'name', {
+    value: 'Error',
+    enumerable: false,
+    writable: true,
+    configurable: true
+  })
+  err.code = code
+  return err
+
+  // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311
+  // so that our error message is identical to the native message.
+  function getMessage(filename, parentPath = null, packageJsonPath = null) {
+    const ext = path.extname(filename)
+    let msg = `Must use import to load ES Module: ${filename}`;
+    if (parentPath && packageJsonPath) {
+      const path = require('path');
+      const basename = path.basename(filename) === path.basename(parentPath) ?
+        filename : path.basename(filename);
+      msg +=
+        '\nrequire() of ES modules is not supported.\nrequire() of ' +
+        `${filename} ${parentPath ? `from ${parentPath} ` : ''}` +
+        `is an ES module file as it is a ${ext} file whose nearest parent ` +
+        `package.json contains "type": "module" which defines all ${ext} ` +
+        'files in that package scope as ES modules.\nInstead ' +
+        'change the requiring code to use ' +
+        'import(), or remove "type": "module" from ' +
+        `${packageJsonPath}.\n`;
+      return msg;
+    }
+    return msg;
+  }
+}

src/esm.ts

@@ -8,7 +8,10 @@ const { createResolve } = require('../dist-raw/node-esm-resolve-implementation')
 
 export function registerAndCreateEsmHooks (opts?: RegisterOptions) {
   // Automatically performs registration just like `-r ts-node/register`
-  const tsNodeInstance = register(opts)
+  const tsNodeInstance = register({
+    ...opts,
+    experimentalEsmLoader: true
+  })
 
   // Custom implementation that considers additional file extensions and automatically adds file extensions
   const nodeResolveImplementation = createResolve({

src/index.spec.ts

@@ -680,12 +680,12 @@ describe('ts-node', function () {
     })
   })
 
-  if (semver.gte(process.version, '13.0.0')) {
-    describe('esm', () => {
-      this.slow(1000)
+  describe('esm', () => {
+    this.slow(1000)
 
-      const cmd = `node --loader ../../esm.mjs`
+    const cmd = `node --loader ts-node/esm.mjs`
 
+    if (semver.gte(process.version, '13.0.0')) {
       it('should compile and execute as ESM', (done) => {
         exec(`${cmd} index.ts`, { cwd: join(__dirname, '../tests/esm') }, function (err, stdout) {
           expect(err).to.equal(null)
@@ -703,6 +703,23 @@ describe('ts-node', function () {
         })
 
       })
+      it('throws ERR_REQUIRE_ESM when attempting to require() an ESM script while ESM loader is enabled', function (done) {
+        exec(`${cmd} ./index.js`, { cwd: join(__dirname, '../tests/esm-err-require-esm') }, function (err, stdout, stderr) {
+          expect(err).to.not.equal(null)
+          expect(stderr).to.contain('Error [ERR_REQUIRE_ESM]: Must use import to load ES Module:')
+
+          return done()
+        })
+      })
+    }
+
+    it('executes ESM as CJS, ignoring package.json "types" field (for backwards compatibility; should be changed in next major release to throw ERR_REQUIRE_ESM)', function (done) {
+      exec(`${BIN_PATH} ./tests/esm-err-require-esm/index.js`, function (err, stdout) {
+        expect(err).to.equal(null)
+        expect(stdout).to.equal('CommonJS\n')
+
+        return done()
+      })
     })
-  }
+  })
 })

src/index.ts

@@ -5,6 +5,25 @@ import { BaseError } from 'make-error'
 import * as util from 'util'
 import * as _ts from 'typescript'
 
+/**
+ * Does this version of node obey the package.json "type" field
+ * and throw ERR_REQUIRE_ESM when attempting to require() an ESM modules.
+ */
+const engineSupportsPackageTypeField = parseInt(process.versions.node.split('.')[0], 10) >= 12
+
+// Loaded conditionally so we don't need to support older node versions
+let assertScriptCanLoadAsCJSImpl: ((filename: string) => void) | undefined
+
+/**
+ * Assert that script can be loaded as CommonJS when we attempt to require it.
+ * If it should be loaded as ESM, throw ERR_REQUIRE_ESM like node does.
+ */
+function assertScriptCanLoadAsCJS (filename: string) {
+  if (!engineSupportsPackageTypeField) return
+  if (!assertScriptCanLoadAsCJSImpl) assertScriptCanLoadAsCJSImpl = require('../dist-raw/node-cjs-loader-utils').assertScriptCanLoadAsCJSImpl
+  assertScriptCanLoadAsCJSImpl!(filename)
+}
+
 /**
  * Registered `ts-node` instance information.
  */
@@ -179,6 +198,12 @@ export interface CreateOptions {
   readFile?: (path: string) => string | undefined
   fileExists?: (path: string) => boolean
   transformers?: _ts.CustomTransformers | ((p: _ts.Program) => _ts.CustomTransformers)
+  /**
+   * True if require() hooks should interop with experimental ESM loader.
+   * Enabled explicitly via a flag since it is a breaking change.
+   * @internal
+   */
+  experimentalEsmLoader?: boolean
 }
 
 /**
@@ -247,7 +272,8 @@ export const DEFAULTS: RegisterOptions = {
   transpileOnly: yn(process.env.TS_NODE_TRANSPILE_ONLY),
   typeCheck: yn(process.env.TS_NODE_TYPE_CHECK),
   compilerHost: yn(process.env.TS_NODE_COMPILER_HOST),
-  logError: yn(process.env.TS_NODE_LOG_ERROR)
+  logError: yn(process.env.TS_NODE_LOG_ERROR),
+  experimentalEsmLoader: false
 }
 
 /**
@@ -770,9 +796,6 @@ export function create (rawOptions: CreateOptions = {}): Register {
     }
   }
 
-  const cannotCompileViaBothCodepathsErrorMessage = 'Cannot compile the same file via both `require()` and ESM hooks codepaths. ' +
-    'This breaks source-map-support, which cannot tell the difference between the two sourcemaps. ' +
-    'To avoid this problem, load each .ts file as only ESM or only CommonJS.'
   // Create a simple TypeScript compiler proxy.
   function compile (code: string, fileName: string, lineOffset = 0) {
     const normalizedFileName = normalizeSlashes(fileName)
@@ -854,6 +877,10 @@ function registerExtension (
   require.extensions[ext] = function (m: any, filename) { // tslint:disable-line
     if (register.ignored(filename)) return old(m, filename)
 
+    if (register.options.experimentalEsmLoader) {
+      assertScriptCanLoadAsCJS(filename)
+    }
+
     const _compile = m._compile
 
     m._compile = function (code: string, fileName: string) {

tests/esm-err-require-esm/esm-package/loaded-as.ts

@@ -0,0 +1,5 @@
+// Log if this file is loaded as ESM or CommonJS
+if(typeof module !== 'undefined')
+  console.log('CommonJS')
+else
+  console.log('ESM')

tests/esm-err-require-esm/esm-package/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

tests/esm-err-require-esm/index.js

@@ -0,0 +1 @@
+require('./esm-package/loaded-as')