Cleanup raw and dist-raw directories (#1717)

* add notes and todo list

* safely repeating changes from add-cjs-loader-resolve to remember what I did and filter any WIP mess I may have forgotten about

* more sync

* fix downloaded artifacts

* sync

* sync

* sync

* sync

* cleanup

* sync node-internal-modules-cjs-loader-old.js with raw

* sync internal/modules/package_json_reader with new naming convention

* sync node-internal-modules-cjs-loader; rename back to old name

* sync

* more sync of filenames

* sync node-internal-errors

* lint-fix

* fix

* fix

* further alignment

* fix

* fix

* upgrade v8-compile-cache-lib to get rid of eventemitter warning in tests

.gitignore

@@ -4,7 +4,7 @@
 coverage/
 .DS_Store
 npm-debug.log
-dist/
+/dist/
 tsconfig.schema.json
 tsconfig.schemastore-schema.json
 .idea/

ava.config.cjs

@@ -24,7 +24,7 @@ module.exports = {
    * ts-node, from either node_modules or tests/node_modules
    */
 
-  const { existsSync } = require('fs');
+  const { existsSync, rmSync } = require('fs');
   const rimraf = require('rimraf');
   const { resolve } = require('path');
 
@@ -35,6 +35,7 @@ module.exports = {
   expect(() => {createRequire(resolve(__dirname, 'tests/foo.js')).resolve('ts-node')}).toThrow();
 
   function remove(p) {
-    if(existsSync(p)) rimraf.sync(p, {recursive: true})
+    // Avoid node deprecation warning triggered by rimraf
+    if(existsSync(p)) (rmSync || rimraf.sync)(p, {recursive: true})
   }
 }

dist-raw/NODE-LICENSE.md

@@ -0,0 +1,24 @@
+This directory contains portions of Node.js source code which is licensed as follows:
+
+---
+
+Copyright Joyent, Inc. and other Node contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.

dist-raw/README.md

@@ -11,3 +11,26 @@ in a factory function, we will not indent the function body, to avoid whitespace
 One obvious problem with this approach: the code has been pulled from one version of node, whereas users of ts-node
 run multiple versions of node.
 Users running node 12 may see that ts-node behaves like node 14, for example.
+
+## `raw` directory
+
+Within the `raw` directory, we keep unmodified copies of the node source files.  This allows us to use diffing tools to
+compare files in `raw` to those in `dist-raw`, which will highlight all of the changes we have made.  Hopefully, these
+changes are as minimal as possible.
+
+## Naming convention
+
+Not used consistently, but the idea is:
+
+`node-<directory>(...-<directory>)-<filename>.js`
+
+`node-internal-errors.js` -> `github.com/nodejs/node/blob/TAG/lib/internal/errors.js`
+
+So, take the path within node's `lib/` directory, and replace slashes with hyphens.
+
+In the `raw` directory, files are suffixed with the version number or revision from which
+they were downloaded.
+
+If they have a `stripped` suffix, this means they have large chunks of code deleted, but no other modifications.
+This is useful when diffing.  Sometimes our `dist-raw` files only have a small part of a much larger node source file.
+It is easier to diff `raw/*-stripped.js` against `dist-raw/*.js`.

dist-raw/node-errors.js

@@ -1,3 +1,5 @@
+// TODO Sync this with node-internal-errors?
+
 exports.codes = {
   ERR_INPUT_TYPE_NOT_ALLOWED: createErrorCtor(joinArgs('ERR_INPUT_TYPE_NOT_ALLOWED')),
   ERR_INVALID_ARG_VALUE: createErrorCtor(joinArgs('ERR_INVALID_ARG_VALUE')),

dist-raw/node-internal-errors.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const path = require('path');
+
+module.exports = {
+  createErrRequireEsm
+};
+
+// Native ERR_REQUIRE_ESM Error is declared here:
+//   https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313
+// Error class factory is implemented here:
+//   function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341
+//   function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278
+// The code below should create an error that matches the native error as closely as possible.
+// Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error.
+function createErrRequireEsm(filename, parentPath, packageJsonPath) {
+  const code = 'ERR_REQUIRE_ESM'
+  const err = new Error(getErrRequireEsmMessage(filename, parentPath, packageJsonPath))
+  // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field.
+  // This trick is copied from node's source.
+  err.name = `Error [${ code }]`
+  err.stack
+  Object.defineProperty(err, 'name', {
+    value: 'Error',
+    enumerable: false,
+    writable: true,
+    configurable: true
+  })
+  err.code = code
+  return err
+}
+
+// Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311
+// so that our error message is identical to the native message.
+function getErrRequireEsmMessage(filename, parentPath = null, packageJsonPath = null) {
+  const ext = path.extname(filename)
+  let msg = `Must use import to load ES Module: ${filename}`;
+  if (parentPath && packageJsonPath) {
+    const path = require('path');
+    const basename = path.basename(filename) === path.basename(parentPath) ?
+      filename : path.basename(filename);
+    msg +=
+      '\nrequire() of ES modules is not supported.\nrequire() of ' +
+      `${filename} ${parentPath ? `from ${parentPath} ` : ''}` +
+      `is an ES module file as it is a ${ext} file whose nearest parent ` +
+      `package.json contains "type": "module" which defines all ${ext} ` +
+      'files in that package scope as ES modules.\nInstead ' +
+      'change the requiring code to use ' +
+      'import(), or remove "type": "module" from ' +
+      `${packageJsonPath}.\n`;
+    return msg;
+  }
+  return msg;
+}

dist-raw/node-internal-modules-cjs-loader.js

@@ -0,0 +1,99 @@
+// Copied from several files in node's source code.
+// https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js
+// Each function and variable below must have a comment linking to the source in node's github repo.
+
+const {
+  JSONParse,
+  SafeMap,
+  StringPrototypeEndsWith,
+  StringPrototypeLastIndexOf,
+  StringPrototypeIndexOf,
+  StringPrototypeSlice,
+} = require('./node-primordials');
+const path = require('path');
+const { sep } = path;
+const packageJsonReader = require('./node-internal-modules-package_json_reader');
+const {normalizeSlashes} = require('../dist/util');
+const {createErrRequireEsm} = require('./node-internal-errors');
+
+
+
+// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249
+const packageJsonCache = new SafeMap();
+
+// Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304
+function readPackage(requestPath) {
+  const jsonPath = path.resolve(requestPath, 'package.json');
+
+  const existing = packageJsonCache.get(jsonPath);
+  if (existing !== undefined) return existing;
+
+  const result = packageJsonReader.read(jsonPath);
+  const json = result.containsKeys === false ? '{}' : result.string;
+  if (json === undefined) {
+    packageJsonCache.set(jsonPath, false);
+    return false;
+  }
+
+  try {
+    const parsed = JSONParse(json);
+    const filtered = {
+      name: parsed.name,
+      main: parsed.main,
+      exports: parsed.exports,
+      imports: parsed.imports,
+      type: parsed.type
+    };
+    packageJsonCache.set(jsonPath, filtered);
+    return filtered;
+  } catch (e) {
+    e.path = jsonPath;
+    e.message = 'Error parsing ' + jsonPath + ': ' + e.message;
+    throw e;
+  }
+}
+
+// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301
+function readPackageScope(checkPath) {
+  const rootSeparatorIndex = StringPrototypeIndexOf(checkPath, sep);
+  let separatorIndex;
+  while (
+    (separatorIndex = StringPrototypeLastIndexOf(checkPath, sep)) > rootSeparatorIndex
+  ) {
+    checkPath = StringPrototypeSlice(checkPath, 0, separatorIndex);
+    if (StringPrototypeEndsWith(checkPath, sep + 'node_modules'))
+      return false;
+    const pjson = readPackage(checkPath);
+    if (pjson) return {
+      path: checkPath,
+      data: pjson
+    };
+  }
+  return false;
+}
+
+/**
+ * copied from Module._extensions['.js']
+ * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120
+ * @param {import('../src/index').Service} service
+ * @param {NodeJS.Module} module
+ * @param {string} filename
+ */
+function assertScriptCanLoadAsCJSImpl(service, module, filename) {
+  const pkg = readPackageScope(filename);
+
+  // ts-node modification: allow our configuration to override
+  const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename));
+  if(tsNodeClassification.moduleType === 'cjs') return;
+
+  // Function require shouldn't be used in ES modules.
+  if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) {
+    const parentPath = module.parent && module.parent.filename;
+    const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null;
+    throw createErrRequireEsm(filename, parentPath, packageJsonPath);
+  }
+}
+
+module.exports = {
+  assertScriptCanLoadAsCJSImpl
+};

dist-raw/node-esm-default-get-format.js → dist-raw/node-internal-modules-esm-get_format.js

@@ -1,7 +1,4 @@
 // Copied from https://raw.githubusercontent.com/nodejs/node/v15.3.0/lib/internal/modules/esm/get_format.js
-// Then modified to suite our needs.
-// Formatting is intentionally bad to keep the diff as small as possible, to make it easier to merge
-// upstream changes and understand our modifications.
 
 'use strict';
 const {
@@ -19,7 +16,7 @@ const experimentalJsonModules =
 const experimentalSpeciferResolution =
   getOptionValue('--experimental-specifier-resolution');
 const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
-const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []});
+const { getPackageType } = require('./node-internal-modules-esm-resolve').createResolve({tsExtensions: [], jsExtensions: []});
 const { URL, fileURLToPath } = require('url');
 const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes;