feat(rosetta): Rosetta manages dependencies automatically

packages/@jsii/spec/lib/configuration.ts

@@ -171,5 +171,9 @@ export interface PackageJson {
    */
   devDependencies?: Record<string, string>;
 
+  bundleDependencies?: string[];
+
+  bundledDependencies?: string[];
+
   [key: string]: unknown;
 }

packages/jsii-calc/package.json

@@ -33,7 +33,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile",
+    "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile --verbose",
     "watch": "jsii --project-references -w",
     "lint": "eslint . --ext .js,.ts --ignore-path=.gitignore",
     "lint:fix": "yarn lint --fix",

packages/jsii-rosetta/README.md

@@ -44,20 +44,14 @@ someObject.someMethod('foo', {
 ### Enforcing correct examples
 
 By default, Rosetta will accept non-compiling examples. If you set
-`jsii.metadata.jsii.rosetta.strict` to `true` in your `package.json`,
+`jsiiRosetta.strict` to `true` in your `package.json`,
 the Rosetta command will fail if any example contains an error:
 
 ```js
 /// package.json
 {
-  "jsii": {
-    "metadata": {
-      "jsii": {
-        "rosetta": {
-          "strict": true
-        }
-      }
-    }
+  "jsiiRosetta": {
+    "strict": true
   }
 }
 ```
@@ -114,6 +108,32 @@ To specify fixtures in an `@example` block, use an accompanying `@exampleMetadat
  */
 ````
 
+### Dependencies
+
+When compiling examples, Rosetta will make sure your package itself and all of
+its `dependencies` and `peerDependencies` are available in the dependency
+closure that your examples will be compiled in.
+
+If there are packages you want to use in an example that should *not* be part
+of your package's dependencies, declare them in `jsiiRosetta.exampleDependencies`
+in your `package.json`:
+
+```js
+/// package.json
+{
+  "jsiiRosetta": {
+    "exampleDependencies": {
+      "@some-other/package": "^1.2.3",
+      "@yet-another/package": "*",
+    }
+  }
+}
+```
+
+You can also set up a directory with correct dependencies yourself, and pass
+`--directory` when running `jsii-rosetta extract`. We recommend using the
+automatic closure building mechanism and specifying `exampleDependencies` though.
+
 ## Rosetta for package publishers
 
 This section describes how Rosetta integrates into your build process.

packages/jsii-rosetta/bin/jsii-rosetta.ts

@@ -210,20 +210,13 @@ function main() {
           args.fail = args.f = true;
         }
 
-        // Easiest way to get a fixed working directory (for sources) in is to
-        // chdir, since underneath the in-memory layer we're using a regular TS
-        // compilerhost. Have to make all file references absolute before we chdir
-        // though.
         const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x));
 
         const absCacheFrom = fmap(args.cache ?? args['cache-from'], path.resolve);
         const absCacheTo = fmap(args.cache ?? args['cache-to'] ?? args.output, path.resolve);
 
-        if (args.directory) {
-          process.chdir(args.directory);
-        }
-
         const extractOptions: ExtractOptions = {
+          compilationDirectory: args.directory,
           includeCompilerDiagnostics: !!args.compile,
           validateAssemblies: args['validate-assemblies'],
           only: args.include,

packages/jsii-rosetta/lib/commands/extract.ts

@@ -44,6 +44,14 @@ export interface ExtractOptions {
    */
   readonly writeToImplicitTablets?: boolean;
 
+  /**
+   * What directory to compile the samples in
+   *
+   * @default - Rosetta manages the compilation directory
+   * @deprecated Samples declare their own dependencies instead
+   */
+  readonly compilationDirectory?: string;
+
   /**
    * Make a translator (just for testing)
    */
@@ -81,7 +89,7 @@ export async function extractSnippets(
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies ?? false);
 
-  let snippets = Array.from(allTypeScriptSnippets(assemblies, options.loose));
+  let snippets = Array.from(await allTypeScriptSnippets(assemblies, options.loose));
   if (only.length > 0) {
     snippets = filterSnippets(snippets, only);
   }

packages/jsii-rosetta/lib/commands/infuse.ts

@@ -75,7 +75,7 @@ export async function infuse(assemblyLocations: string[], options?: InfuseOption
   }
   availableTranslations.addTablets(...Object.values(defaultTablets));
 
-  const { translationsByFqn, originalsByKey } = availableSnippetsPerFqn(assemblies, availableTranslations);
+  const { translationsByFqn, originalsByKey } = await availableSnippetsPerFqn(assemblies, availableTranslations);
 
   const additionalOutputTablet = options?.cacheToFile
     ? await LanguageTablet.fromOptionalFile(options?.cacheToFile)
@@ -244,10 +244,10 @@ function insertExample(
  *
  * Returns a map of fqns to a list of keys that represent snippets that include the fqn.
  */
-function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) {
+async function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) {
   const ret = new DefaultRecord<TranslatedSnippet>();
 
-  const originalsByKey = indexBy(allTypeScriptSnippets(asms), snippetKey);
+  const originalsByKey = indexBy(await allTypeScriptSnippets(asms), snippetKey);
 
   const translations = Object.keys(originalsByKey)
     .map((key) => translationsTablet.tryGetSnippet(key))

packages/jsii-rosetta/lib/commands/transliterate.ts

@@ -9,7 +9,7 @@ import { debug } from '../logging';
 import { RosettaTabletReader, UnknownSnippetMode } from '../rosetta-reader';
 import { SnippetParameters, typeScriptSnippetFromVisibleSource, ApiLocation, parseMetadataLine } from '../snippet';
 import { Translation } from '../tablets/tablets';
-import { fmap } from '../util';
+import { fmap, Mutable } from '../util';
 import { extractSnippets } from './extract';
 
 export interface TransliterateAssemblyOptions {
@@ -143,7 +143,6 @@ async function loadAssemblies(
   return result;
 }
 
-type Mutable<T> = { -readonly [K in keyof T]: Mutable<T[K]> };
 type AssemblyLoader = () => Promise<Mutable<Assembly>>;
 
 function prefixDisclaimer(translation: Translation): string {

packages/jsii-rosetta/lib/commands/trim-cache.ts

@@ -20,7 +20,7 @@ export async function trimCache(options: TrimCacheOptions): Promise<void> {
   logging.info(`Loading ${options.assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(options.assemblyLocations, false);
 
-  const snippets = Array.from(allTypeScriptSnippets(assemblies));
+  const snippets = Array.from(await allTypeScriptSnippets(assemblies));
 
   const original = await LanguageTablet.fromFile(options.cacheFile);
   const updated = new LanguageTablet();

packages/jsii-rosetta/lib/find-utils.ts

@@ -0,0 +1,86 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+/**
+ * Find the directory that contains a given dependency, identified by its 'package.json', from a starting search directory
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export async function findDependencyDirectory(dependencyName: string, searchStart: string) {
+  // Explicitly do not use 'require("dep/package.json")' because that will fail if the
+  // package does not export that particular file.
+  const entryPoint = require.resolve(dependencyName, {
+    paths: [searchStart],
+  });
+
+  // Search up from the given directory, looking for a package.json that matches
+  // the dependency name (so we don't accidentally find stray 'package.jsons').
+  const depPkgJsonPath = await findPackageJsonUp(dependencyName, path.dirname(entryPoint));
+
+  if (!depPkgJsonPath) {
+    throw new Error(`Could not find dependency '${dependencyName}' from '${searchStart}'`);
+  }
+
+  return depPkgJsonPath;
+}
+
+/**
+ * Find the package.json for a given package upwards from the given directory
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export async function findPackageJsonUp(packageName: string, directory: string) {
+  return findUp(directory, async (dir) => {
+    const pjFile = path.join(dir, 'package.json');
+    return (await fs.pathExists(pjFile)) && (await fs.readJson(pjFile)).name === packageName;
+  });
+}
+
+/**
+ * Find a directory up the tree from a starting directory matching a condition
+ *
+ * Will return `undefined` if no directory matches
+ *
+ * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all
+ * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236)
+ */
+export function findUp(directory: string, pred: (dir: string) => Promise<boolean>): Promise<string | undefined>;
+export function findUp(directory: string, pred: (dir: string) => boolean): string | undefined;
+// eslint-disable-next-line @typescript-eslint/promise-function-async
+export function findUp(
+  directory: string,
+  pred: ((dir: string) => boolean) | ((dir: string) => Promise<boolean>),
+): Promise<string | undefined> | string | undefined {
+  const result = pred(directory);
+  if (isPromise(result)) {
+    return result.then((thisDirectory) => (thisDirectory ? directory : recurse()));
+  }
+
+  return result ? directory : recurse();
+
+  function recurse() {
+    const parent = path.dirname(directory);
+    if (parent === directory) {
+      return undefined;
+    }
+    return findUp(parent, pred as any);
+  }
+}
+
+function isPromise<A>(x: A | Promise<A>): x is Promise<A> {
+  return typeof x === 'object' && (x as any).then;
+}
+
+/**
+ * Whether the given dependency is a built-in
+ *
+ * Some dependencies that occur in `package.json` are also built-ins in modern Node
+ * versions (most egregious example: 'punycode'). Detect those and filter them out.
+ */
+export function isBuiltinModule(depName: string) {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires
+  const { builtinModules } = require('module');
+  return (builtinModules ?? []).includes(depName);
+}