feat(rosetta): Rosetta manages dependencies automatically (#3269)

It's quite an annoying bother and pretty brittle to have to set up a full dependency directory to compile examples in.

In CDK, we use the directory of the `decdk` package to do this, because we have external knowledge that all packages
will be symlinked into this directory.

That mechanism is brittle and revolves around having outside knowledge. Instead, make Rosetta manage the dependency closure itself, when user declare `exampleDependencies` in `package.json`:

```
{
  "jsiiRosetta": {
    "exampleDependencies": {
      "@some-other/package": "^1.2.3",
      "@yet-another/package": "*",
    }
  }
}
```

It is not necessary to redeclare any of the `dependencies` and `peerDependencies` already in `package.json`: those are automatically included.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

packages/@jsii/spec/lib/configuration.ts

@@ -171,5 +171,9 @@ export interface PackageJson {
    */
   devDependencies?: Record<string, string>;
 
+  bundleDependencies?: string[];
+
+  bundledDependencies?: string[];
+
   [key: string]: unknown;
 }

packages/jsii-calc/package.json

@@ -33,7 +33,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile",
+    "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile --verbose",
     "watch": "jsii --project-references -w",
     "lint": "eslint . --ext .js,.ts --ignore-path=.gitignore",
     "lint:fix": "yarn lint --fix",

packages/jsii-rosetta/README.md

@@ -44,20 +44,14 @@ someObject.someMethod('foo', {
 ### Enforcing correct examples
 
 By default, Rosetta will accept non-compiling examples. If you set
-`jsii.metadata.jsii.rosetta.strict` to `true` in your `package.json`,
+`jsiiRosetta.strict` to `true` in your `package.json`,
 the Rosetta command will fail if any example contains an error:
 
 ```js
 /// package.json
 {
-  "jsii": {
-    "metadata": {
-      "jsii": {
-        "rosetta": {
-          "strict": true
-        }
-      }
-    }
+  "jsiiRosetta": {
+    "strict": true
   }
 }
 ```
@@ -114,6 +108,32 @@ To specify fixtures in an `@example` block, use an accompanying `@exampleMetadat
  */
 ````
 
+### Dependencies
+
+When compiling examples, Rosetta will make sure your package itself and all of
+its `dependencies` and `peerDependencies` are available in the dependency
+closure that your examples will be compiled in.
+
+If there are packages you want to use in an example that should *not* be part
+of your package's dependencies, declare them in `jsiiRosetta.exampleDependencies`
+in your `package.json`:
+
+```js
+/// package.json
+{
+  "jsiiRosetta": {
+    "exampleDependencies": {
+      "@some-other/package": "^1.2.3",
+      "@yet-another/package": "*",
+    }
+  }
+}
+```
+
+You can also set up a directory with correct dependencies yourself, and pass
+`--directory` when running `jsii-rosetta extract`. We recommend using the
+automatic closure building mechanism and specifying `exampleDependencies` though.
+
 ## Rosetta for package publishers
 
 This section describes how Rosetta integrates into your build process.

packages/jsii-rosetta/bin/jsii-rosetta.ts

@@ -238,20 +238,13 @@ function main() {
           args.fail = args.f = true;
         }
 
-        // Easiest way to get a fixed working directory (for sources) in is to
-        // chdir, since underneath the in-memory layer we're using a regular TS
-        // compilerhost. Have to make all file references absolute before we chdir
-        // though.
         const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x));
 
         const absCacheFrom = fmap(args.cache ?? args['cache-from'], path.resolve);
         const absCacheTo = fmap(args.cache ?? args['cache-to'] ?? args.output, path.resolve);
 
-        if (args.directory) {
-          process.chdir(args.directory);
-        }
-
         const extractOptions: ExtractOptions = {
+          compilationDirectory: args.directory,
           includeCompilerDiagnostics: !!args.compile,
           validateAssemblies: args['validate-assemblies'],
           only: args.include,

packages/jsii-rosetta/lib/commands/coverage.ts

@@ -7,7 +7,7 @@ export async function checkCoverage(assemblyLocations: readonly string[]): Promi
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(assemblyLocations, false);
 
-  const snippets = Array.from(allTypeScriptSnippets(assemblies, true));
+  const snippets = Array.from(await allTypeScriptSnippets(assemblies, true));
 
   const translator = new RosettaTranslator({
     assemblies: assemblies.map((a) => a.assembly),

packages/jsii-rosetta/lib/commands/extract.ts

@@ -44,6 +44,14 @@ export interface ExtractOptions {
    */
   readonly writeToImplicitTablets?: boolean;
 
+  /**
+   * What directory to compile the samples in
+   *
+   * @default - Rosetta manages the compilation directory
+   * @deprecated Samples declare their own dependencies instead
+   */
+  readonly compilationDirectory?: string;
+
   /**
    * Make a translator (just for testing)
    */
@@ -88,7 +96,7 @@ export async function extractSnippets(
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies ?? false);
 
-  let snippets = Array.from(allTypeScriptSnippets(assemblies, options.loose));
+  let snippets = Array.from(await allTypeScriptSnippets(assemblies, options.loose));
   if (only.length > 0) {
     snippets = filterSnippets(snippets, only);
   }

packages/jsii-rosetta/lib/commands/infuse.ts

@@ -75,7 +75,7 @@ export async function infuse(assemblyLocations: string[], options?: InfuseOption
   }
   availableTranslations.addTablets(...Object.values(defaultTablets));
 
-  const { translationsByFqn, originalsByKey } = availableSnippetsPerFqn(assemblies, availableTranslations);
+  const { translationsByFqn, originalsByKey } = await availableSnippetsPerFqn(assemblies, availableTranslations);
 
   const additionalOutputTablet = options?.cacheToFile
     ? await LanguageTablet.fromOptionalFile(options?.cacheToFile)
@@ -244,10 +244,10 @@ function insertExample(
  *
  * Returns a map of fqns to a list of keys that represent snippets that include the fqn.
  */
-function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) {
+async function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) {
   const ret = new DefaultRecord<TranslatedSnippet>();
 
-  const originalsByKey = indexBy(allTypeScriptSnippets(asms), snippetKey);
+  const originalsByKey = indexBy(await allTypeScriptSnippets(asms), snippetKey);
 
   const translations = Object.keys(originalsByKey)
     .map((key) => translationsTablet.tryGetSnippet(key))

packages/jsii-rosetta/lib/commands/transliterate.ts

@@ -6,6 +6,7 @@ import { TargetLanguage } from '../languages';
 import { debug } from '../logging';
 import { RosettaTabletReader, UnknownSnippetMode } from '../rosetta-reader';
 import { typeScriptSnippetFromVisibleSource, ApiLocation } from '../snippet';
+import { Mutable } from '../util';
 import { extractSnippets } from './extract';
 
 export interface TransliterateAssemblyOptions {
@@ -136,7 +137,6 @@ async function loadAssemblies(
   return result;
 }
 
-type Mutable<T> = { -readonly [K in keyof T]: Mutable<T[K]> };
 type AssemblyLoader = () => Promise<Mutable<Assembly>>;
 
 function transliterateType(type: Type, rosetta: RosettaTabletReader, language: TargetLanguage): void {

packages/jsii-rosetta/lib/commands/trim-cache.ts

@@ -20,7 +20,7 @@ export async function trimCache(options: TrimCacheOptions): Promise<void> {
   logging.info(`Loading ${options.assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(options.assemblyLocations, false);
 
-  const snippets = Array.from(allTypeScriptSnippets(assemblies));
+  const snippets = Array.from(await allTypeScriptSnippets(assemblies));
 
   const original = await LanguageTablet.fromFile(options.cacheFile);
   const updated = new LanguageTablet();

packages/jsii-rosetta/lib/find-utils.ts

@@ -0,0 +1,82 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+/**
+ * Find the directory that contains a given dependency, identified by its 'package.json', from a starting search directory
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export async function findDependencyDirectory(dependencyName: string, searchStart: string) {
+  // Explicitly do not use 'require("dep/package.json")' because that will fail if the
+  // package does not export that particular file.
+  const entryPoint = require.resolve(dependencyName, {
+    paths: [searchStart],
+  });
+
+  // Search up from the given directory, looking for a package.json that matches
+  // the dependency name (so we don't accidentally find stray 'package.jsons').
+  const depPkgJsonPath = await findPackageJsonUp(dependencyName, path.dirname(entryPoint));
+
+  if (!depPkgJsonPath) {
+    throw new Error(`Could not find dependency '${dependencyName}' from '${searchStart}'`);
+  }
+
+  return depPkgJsonPath;
+}
+
+/**
+ * Find the package.json for a given package upwards from the given directory
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export async function findPackageJsonUp(packageName: string, directory: string) {
+  return findUp(directory, async (dir) => {
+    const pjFile = path.join(dir, 'package.json');
+    return (await fs.pathExists(pjFile)) && (await fs.readJson(pjFile)).name === packageName;
+  });
+}
+
+/**
+ * Find a directory up the tree from a starting directory matching a condition
+ *
+ * Will return `undefined` if no directory matches
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export function findUp(directory: string, pred: (dir: string) => Promise<boolean>): Promise<string | undefined>;
+export function findUp(directory: string, pred: (dir: string) => boolean): string | undefined;
+// eslint-disable-next-line @typescript-eslint/promise-function-async
+export function findUp(
+  directory: string,
+  pred: (dir: string) => boolean | Promise<boolean>,
+): Promise<string | undefined> | string | undefined {
+  const result = pred(directory);
+  if (result instanceof Promise) {
+    return result.then((thisDirectory) => (thisDirectory ? directory : recurse()));
+  }
+
+  return result ? directory : recurse();
+
+  function recurse() {
+    const parent = path.dirname(directory);
+    if (parent === directory) {
+      return undefined;
+    }
+    return findUp(parent, pred as any);
+  }
+}
+
+/**
+ * Whether the given dependency is a built-in
+ *
+ * Some dependencies that occur in `package.json` are also built-ins in modern Node
+ * versions (most egregious example: 'punycode'). Detect those and filter them out.
+ */
+export function isBuiltinModule(depName: string) {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires
+  const { builtinModules } = require('module');
+  return (builtinModules ?? []).includes(depName);
+}