diff --git a/packages/@jsii/spec/lib/configuration.ts b/packages/@jsii/spec/lib/configuration.ts index ca5e9fffec..16355fea82 100644 --- a/packages/@jsii/spec/lib/configuration.ts +++ b/packages/@jsii/spec/lib/configuration.ts @@ -171,5 +171,9 @@ export interface PackageJson { */ devDependencies?: Record; + bundleDependencies?: string[]; + + bundledDependencies?: string[]; + [key: string]: unknown; } diff --git a/packages/jsii-calc/package.json b/packages/jsii-calc/package.json index 463dbf860a..452e158d81 100644 --- a/packages/jsii-calc/package.json +++ b/packages/jsii-calc/package.json @@ -33,7 +33,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile", + "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile --verbose", "watch": "jsii --project-references -w", "lint": "eslint . --ext .js,.ts --ignore-path=.gitignore", "lint:fix": "yarn lint --fix", diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md index 7b1e32c7a8..6139d4b9cb 100644 --- a/packages/jsii-rosetta/README.md +++ b/packages/jsii-rosetta/README.md @@ -44,20 +44,14 @@ someObject.someMethod('foo', { ### Enforcing correct examples By default, Rosetta will accept non-compiling examples. If you set -`jsii.metadata.jsii.rosetta.strict` to `true` in your `package.json`, +`jsiiRosetta.strict` to `true` in your `package.json`, the Rosetta command will fail if any example contains an error: ```js /// package.json { - "jsii": { - "metadata": { - "jsii": { - "rosetta": { - "strict": true - } - } - } + "jsiiRosetta": { + "strict": true } } ``` @@ -114,6 +108,32 @@ To specify fixtures in an `@example` block, use an accompanying `@exampleMetadat */ ```` +### Dependencies + +When compiling examples, Rosetta will make sure your package itself and all of +its `dependencies` and `peerDependencies` are available in the dependency +closure that your examples will be compiled in. + +If there are packages you want to use in an example that should *not* be part +of your package's dependencies, declare them in `jsiiRosetta.exampleDependencies` +in your `package.json`: + +```js +/// package.json +{ + "jsiiRosetta": { + "exampleDependencies": { + "@some-other/package": "^1.2.3", + "@yet-another/package": "*", + } + } +} +``` + +You can also set up a directory with correct dependencies yourself, and pass +`--directory` when running `jsii-rosetta extract`. We recommend using the +automatic closure building mechanism and specifying `exampleDependencies` though. + ## Rosetta for package publishers This section describes how Rosetta integrates into your build process. diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts index 39df072af2..0d2948ac0e 100644 --- a/packages/jsii-rosetta/bin/jsii-rosetta.ts +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -238,20 +238,13 @@ function main() { args.fail = args.f = true; } - // Easiest way to get a fixed working directory (for sources) in is to - // chdir, since underneath the in-memory layer we're using a regular TS - // compilerhost. Have to make all file references absolute before we chdir - // though. const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x)); const absCacheFrom = fmap(args.cache ?? args['cache-from'], path.resolve); const absCacheTo = fmap(args.cache ?? args['cache-to'] ?? args.output, path.resolve); - if (args.directory) { - process.chdir(args.directory); - } - const extractOptions: ExtractOptions = { + compilationDirectory: args.directory, includeCompilerDiagnostics: !!args.compile, validateAssemblies: args['validate-assemblies'], only: args.include, diff --git a/packages/jsii-rosetta/lib/commands/coverage.ts b/packages/jsii-rosetta/lib/commands/coverage.ts index d7fb57b0d9..d223e41262 100644 --- a/packages/jsii-rosetta/lib/commands/coverage.ts +++ b/packages/jsii-rosetta/lib/commands/coverage.ts @@ -7,7 +7,7 @@ export async function checkCoverage(assemblyLocations: readonly string[]): Promi logging.info(`Loading ${assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(assemblyLocations, false); - const snippets = Array.from(allTypeScriptSnippets(assemblies, true)); + const snippets = Array.from(await allTypeScriptSnippets(assemblies, true)); const translator = new RosettaTranslator({ assemblies: assemblies.map((a) => a.assembly), diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index 2d0105af59..949c2007e7 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -44,6 +44,14 @@ export interface ExtractOptions { */ readonly writeToImplicitTablets?: boolean; + /** + * What directory to compile the samples in + * + * @default - Rosetta manages the compilation directory + * @deprecated Samples declare their own dependencies instead + */ + readonly compilationDirectory?: string; + /** * Make a translator (just for testing) */ @@ -88,7 +96,7 @@ export async function extractSnippets( logging.info(`Loading ${assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies ?? false); - let snippets = Array.from(allTypeScriptSnippets(assemblies, options.loose)); + let snippets = Array.from(await allTypeScriptSnippets(assemblies, options.loose)); if (only.length > 0) { snippets = filterSnippets(snippets, only); } diff --git a/packages/jsii-rosetta/lib/commands/infuse.ts b/packages/jsii-rosetta/lib/commands/infuse.ts index 4820309c9e..183977b3d8 100644 --- a/packages/jsii-rosetta/lib/commands/infuse.ts +++ b/packages/jsii-rosetta/lib/commands/infuse.ts @@ -75,7 +75,7 @@ export async function infuse(assemblyLocations: string[], options?: InfuseOption } availableTranslations.addTablets(...Object.values(defaultTablets)); - const { translationsByFqn, originalsByKey } = availableSnippetsPerFqn(assemblies, availableTranslations); + const { translationsByFqn, originalsByKey } = await availableSnippetsPerFqn(assemblies, availableTranslations); const additionalOutputTablet = options?.cacheToFile ? await LanguageTablet.fromOptionalFile(options?.cacheToFile) @@ -244,10 +244,10 @@ function insertExample( * * Returns a map of fqns to a list of keys that represent snippets that include the fqn. */ -function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) { +async function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) { const ret = new DefaultRecord(); - const originalsByKey = indexBy(allTypeScriptSnippets(asms), snippetKey); + const originalsByKey = indexBy(await allTypeScriptSnippets(asms), snippetKey); const translations = Object.keys(originalsByKey) .map((key) => translationsTablet.tryGetSnippet(key)) diff --git a/packages/jsii-rosetta/lib/commands/transliterate.ts b/packages/jsii-rosetta/lib/commands/transliterate.ts index ea6904a683..5878c1d9ad 100644 --- a/packages/jsii-rosetta/lib/commands/transliterate.ts +++ b/packages/jsii-rosetta/lib/commands/transliterate.ts @@ -6,6 +6,7 @@ import { TargetLanguage } from '../languages'; import { debug } from '../logging'; import { RosettaTabletReader, UnknownSnippetMode } from '../rosetta-reader'; import { typeScriptSnippetFromVisibleSource, ApiLocation } from '../snippet'; +import { Mutable } from '../util'; import { extractSnippets } from './extract'; export interface TransliterateAssemblyOptions { @@ -136,7 +137,6 @@ async function loadAssemblies( return result; } -type Mutable = { -readonly [K in keyof T]: Mutable }; type AssemblyLoader = () => Promise>; function transliterateType(type: Type, rosetta: RosettaTabletReader, language: TargetLanguage): void { diff --git a/packages/jsii-rosetta/lib/commands/trim-cache.ts b/packages/jsii-rosetta/lib/commands/trim-cache.ts index f22ce4aa81..de87328489 100644 --- a/packages/jsii-rosetta/lib/commands/trim-cache.ts +++ b/packages/jsii-rosetta/lib/commands/trim-cache.ts @@ -20,7 +20,7 @@ export async function trimCache(options: TrimCacheOptions): Promise { logging.info(`Loading ${options.assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(options.assemblyLocations, false); - const snippets = Array.from(allTypeScriptSnippets(assemblies)); + const snippets = Array.from(await allTypeScriptSnippets(assemblies)); const original = await LanguageTablet.fromFile(options.cacheFile); const updated = new LanguageTablet(); diff --git a/packages/jsii-rosetta/lib/find-utils.ts b/packages/jsii-rosetta/lib/find-utils.ts new file mode 100644 index 0000000000..77c29907cd --- /dev/null +++ b/packages/jsii-rosetta/lib/find-utils.ts @@ -0,0 +1,82 @@ +import * as fs from 'fs-extra'; +import * as path from 'path'; + +/** + * Find the directory that contains a given dependency, identified by its 'package.json', from a starting search directory + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export async function findDependencyDirectory(dependencyName: string, searchStart: string) { + // Explicitly do not use 'require("dep/package.json")' because that will fail if the + // package does not export that particular file. + const entryPoint = require.resolve(dependencyName, { + paths: [searchStart], + }); + + // Search up from the given directory, looking for a package.json that matches + // the dependency name (so we don't accidentally find stray 'package.jsons'). + const depPkgJsonPath = await findPackageJsonUp(dependencyName, path.dirname(entryPoint)); + + if (!depPkgJsonPath) { + throw new Error(`Could not find dependency '${dependencyName}' from '${searchStart}'`); + } + + return depPkgJsonPath; +} + +/** + * Find the package.json for a given package upwards from the given directory + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export async function findPackageJsonUp(packageName: string, directory: string) { + return findUp(directory, async (dir) => { + const pjFile = path.join(dir, 'package.json'); + return (await fs.pathExists(pjFile)) && (await fs.readJson(pjFile)).name === packageName; + }); +} + +/** + * Find a directory up the tree from a starting directory matching a condition + * + * Will return `undefined` if no directory matches + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export function findUp(directory: string, pred: (dir: string) => Promise): Promise; +export function findUp(directory: string, pred: (dir: string) => boolean): string | undefined; +// eslint-disable-next-line @typescript-eslint/promise-function-async +export function findUp( + directory: string, + pred: (dir: string) => boolean | Promise, +): Promise | string | undefined { + const result = pred(directory); + if (result instanceof Promise) { + return result.then((thisDirectory) => (thisDirectory ? directory : recurse())); + } + + return result ? directory : recurse(); + + function recurse() { + const parent = path.dirname(directory); + if (parent === directory) { + return undefined; + } + return findUp(parent, pred as any); + } +} + +/** + * Whether the given dependency is a built-in + * + * Some dependencies that occur in `package.json` are also built-ins in modern Node + * versions (most egregious example: 'punycode'). Detect those and filter them out. + */ +export function isBuiltinModule(depName: string) { + // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires + const { builtinModules } = require('module'); + return (builtinModules ?? []).includes(depName); +} diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts index 68c63cd855..f9edbb06a6 100644 --- a/packages/jsii-rosetta/lib/jsii/assemblies.ts +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -3,6 +3,7 @@ import * as crypto from 'crypto'; import * as fs from 'fs-extra'; import * as path from 'path'; +import { findDependencyDirectory, isBuiltinModule } from '../find-utils'; import { fixturize } from '../fixtures'; import { extractTypescriptSnippetsFromMarkdown } from '../markdown/extract-snippets'; import { @@ -12,6 +13,7 @@ import { SnippetParameters, ApiLocation, parseMetadataLine, + CompilationDependency, INITIALIZER_METHOD_NAME, } from '../snippet'; import { enforcesStrictMode } from '../strict'; @@ -37,9 +39,17 @@ const sortJson = require('sort-json'); */ export const EXAMPLE_METADATA_JSDOCTAG = 'exampleMetadata'; +interface RosettaPackageJson extends spec.PackageJson { + readonly jsiiRosetta?: { + readonly strict?: boolean; + readonly exampleDependencies?: Record; + }; +} + export interface LoadedAssembly { readonly assembly: spec.Assembly; readonly directory: string; + readonly packageJson?: RosettaPackageJson; } /** @@ -57,10 +67,15 @@ export async function loadAssemblies( return loadAssembly(path.join(location, '.jsii')); } - return { - assembly: await loadAssemblyFromFile(location, validateAssemblies), - directory: path.dirname(location), - }; + const directory = path.dirname(location); + const pjLocation = path.join(directory, 'package.json'); + + const [assembly, packageJson] = await Promise.all([ + loadAssemblyFromFile(location, validateAssemblies), + (await fs.pathExists(pjLocation)) ? fs.readJSON(pjLocation, { encoding: 'utf-8' }) : Promise.resolve(undefined), + ]); + + return { assembly, directory, packageJson }; } } @@ -171,36 +186,50 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc } } -export function allTypeScriptSnippets(assemblies: readonly LoadedAssembly[], loose = false): TypeScriptSnippet[] { - const ret = new Array(); - - for (const { assembly, directory } of assemblies) { - const strict = enforcesStrictMode(assembly); - for (const source of allSnippetSources(assembly)) { - switch (source.type) { - case 'example': - // If an example is an infused example, we do not care about compiler errors. - // We are relying on the tablet cache to have this example stored already. - const [strictForExample, looseForExample] = - source.metadata?.infused !== undefined ? [false, true] : [strict, loose]; - const location = { api: source.location, field: { field: 'example' } } as const; - const snippet = updateParameters(typeScriptSnippetFromSource(source.source, location, strictForExample), { - [SnippetParameters.$PROJECT_DIRECTORY]: directory, - ...source.metadata, - }); - ret.push(fixturize(snippet, looseForExample)); - break; - case 'markdown': - for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.location, strict)) { - const withDirectory = updateParameters(snippet, { - [SnippetParameters.$PROJECT_DIRECTORY]: directory, - }); - ret.push(fixturize(withDirectory, loose)); - } - } - } - } - return ret; +export async function allTypeScriptSnippets( + assemblies: readonly LoadedAssembly[], + loose = false, +): Promise { + return Promise.all( + assemblies + .flatMap((loaded) => allSnippetSources(loaded.assembly).map((source) => ({ source, loaded }))) + .flatMap(({ source, loaded }) => { + switch (source.type) { + case 'example': + return [ + { + snippet: updateParameters( + typeScriptSnippetFromSource( + source.source, + { api: source.location, field: { field: 'example' } }, + isStrict(loaded), + ), + source.metadata ?? {}, + ), + loaded, + }, + ]; + case 'markdown': + return extractTypescriptSnippetsFromMarkdown(source.markdown, source.location, isStrict(loaded)).map( + (snippet) => ({ snippet, loaded }), + ); + } + }) + .map(async ({ snippet, loaded }) => { + const isInfused = snippet.parameters?.infused != null; + + // Ignore fixturization errors if requested on this command, or if the snippet was infused + const ignoreFixtureErrors = loose || isInfused; + + // Also if the snippet was infused: switch off 'strict' mode if it was set + if (isInfused) { + snippet = { ...snippet, strict: false }; + } + + snippet = await withDependencies(loaded, withProjectDirectory(loaded.directory, snippet)); + return fixturize(snippet, ignoreFixtureErrors); + }), + ); } /** @@ -318,3 +347,75 @@ export function findContainingSubmodule(assembly: spec.Assembly, fqn: string): s } return undefined; } + +function withProjectDirectory(dir: string, snippet: TypeScriptSnippet) { + return updateParameters(snippet, { + [SnippetParameters.$PROJECT_DIRECTORY]: dir, + }); +} + +/** + * Return a TypeScript snippet with dependencies added + * + * The dependencies will be taken from the package.json, and will consist of: + * + * - The package itself + * - The package's dependencies and peerDependencies + * - Any additional dependencies declared in `jsiiRosetta.exampleDependencies`. + */ +async function withDependencies(asm: LoadedAssembly, snippet: TypeScriptSnippet): Promise { + const compilationDependencies: Record = {}; + + compilationDependencies[asm.assembly.name] = { + type: 'concrete', + resolvedDirectory: await fs.realpath(asm.directory), + }; + + Object.assign( + compilationDependencies, + mkDict( + await Promise.all( + Object.keys({ ...asm.packageJson?.dependencies, ...asm.packageJson?.peerDependencies }) + .filter((name) => !isBuiltinModule(name)) + .filter( + (name) => + !asm.packageJson?.bundledDependencies?.includes(name) && + !asm.packageJson?.bundleDependencies?.includes(name), + ) + .map( + async (name) => + [ + name, + { + type: 'concrete', + resolvedDirectory: await fs.realpath(await findDependencyDirectory(name, asm.directory)), + }, + ] as const, + ), + ), + ), + ); + + Object.assign( + compilationDependencies, + mkDict( + Object.entries(asm.packageJson?.jsiiRosetta?.exampleDependencies ?? {}).map( + ([name, versionRange]) => [name, { type: 'symbolic', versionRange }] as const, + ), + ), + ); + + return { + ...snippet, + compilationDependencies, + }; +} + +/** + * Whether samples in the assembly should be treated as strict + * + * True if the strict flag is found in the package.json (modern) or the assembly itself (legacy). + */ +function isStrict(loaded: LoadedAssembly) { + return loaded.packageJson?.jsiiRosetta?.strict ?? enforcesStrictMode(loaded.assembly); +} diff --git a/packages/jsii-rosetta/lib/jsii/packages.ts b/packages/jsii-rosetta/lib/jsii/packages.ts index 9aa5e2ec61..b1daae3fa6 100644 --- a/packages/jsii-rosetta/lib/jsii/packages.ts +++ b/packages/jsii-rosetta/lib/jsii/packages.ts @@ -1,45 +1,4 @@ import * as spec from '@jsii/spec'; -import * as fs from 'fs'; -import * as path from 'path'; - -/** - * Resolve a package name in an example to a JSII assembly - * - * We assume we've changed directory to the directory where we need to resolve from. - */ -export function resolvePackage(packageName: string) { - try { - const resolved = require.resolve(`${packageName}/package.json`, { - paths: [process.cwd()], - }); - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(resolved); - } catch { - return undefined; - } -} - -/** - * Find an enclosing package.json file given a filename - * - * Will return `undefined` if a package.json could not be found. - */ -export function findPackageJson(fileName: string) { - // eslint-disable-next-line no-constant-condition - while (true) { - const candidatePath = path.join(fileName, 'package.json'); - if (fs.existsSync(candidatePath)) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(path.resolve(candidatePath)); - } - - const parent = path.dirname(fileName); - if (parent === fileName) { - return undefined; - } - fileName = parent; - } -} export function jsiiTargetParameter(target: spec.Targetable, field: string) { const path = field.split('.'); diff --git a/packages/jsii-rosetta/lib/rosetta-reader.ts b/packages/jsii-rosetta/lib/rosetta-reader.ts index 77f981c800..f8e4b80d0a 100644 --- a/packages/jsii-rosetta/lib/rosetta-reader.ts +++ b/packages/jsii-rosetta/lib/rosetta-reader.ts @@ -162,7 +162,7 @@ export class RosettaTabletReader { // Inventarize the snippets from this assembly, but only if there's a chance // we're going to need them. if (this.unknownSnippets === UnknownSnippetMode.TRANSLATE) { - for (const tsnip of allTypeScriptSnippets([{ assembly, directory: assemblyDir }], this.loose)) { + for (const tsnip of await allTypeScriptSnippets([{ assembly, directory: assemblyDir }], this.loose)) { this.extractedSnippets.set(snippetKey(tsnip), tsnip); } } diff --git a/packages/jsii-rosetta/lib/rosetta-translator.ts b/packages/jsii-rosetta/lib/rosetta-translator.ts index 047db93de0..e1914165f4 100644 --- a/packages/jsii-rosetta/lib/rosetta-translator.ts +++ b/packages/jsii-rosetta/lib/rosetta-translator.ts @@ -1,9 +1,11 @@ import * as spec from '@jsii/spec'; +import * as fs from 'fs-extra'; import { TypeFingerprinter } from './jsii/fingerprinting'; import { TARGET_LANGUAGES } from './languages'; import * as logging from './logging'; import { TypeScriptSnippet, completeSource } from './snippet'; +import { collectDependencies, validateAvailableDependencies, prepareDependencyDirectory } from './snippet-dependencies'; import { snippetKey } from './tablets/key'; import { LanguageTablet, TranslatedSnippet } from './tablets/tablets'; import { translateAll, TranslateAllResult } from './translate_all'; @@ -154,14 +156,49 @@ export class RosettaTranslator { }; } - public async translateAll(snippets: TypeScriptSnippet[], addToTablet = true): Promise { - const result = await translateAll(snippets, this.includeCompilerDiagnostics); + public async translateAll(snippets: TypeScriptSnippet[], addToTablet?: boolean): Promise; + public async translateAll(snippets: TypeScriptSnippet[], options?: TranslateAllOptions): Promise; + public async translateAll( + snippets: TypeScriptSnippet[], + optionsOrAddToTablet?: boolean | TranslateAllOptions, + ): Promise { + const options = + optionsOrAddToTablet && typeof optionsOrAddToTablet === 'object' + ? optionsOrAddToTablet + : { addToTablet: optionsOrAddToTablet }; + + const exampleDependencies = collectDependencies(snippets); + + let compilationDirectory; + let cleanCompilationDir = false; + if (options?.compilationDirectory) { + // If the user provided a directory, we're going to trust-but-confirm. + await validateAvailableDependencies(options.compilationDirectory, exampleDependencies); + compilationDirectory = options.compilationDirectory; + } else { + compilationDirectory = await prepareDependencyDirectory(exampleDependencies); + cleanCompilationDir = true; + } + + const origDir = process.cwd(); + // Easiest way to get a fixed working directory (for sources) in is to chdir + process.chdir(compilationDirectory); + + let result; + try { + result = await translateAll(snippets, this.includeCompilerDiagnostics); + } finally { + process.chdir(origDir); + if (cleanCompilationDir) { + await fs.remove(compilationDirectory); + } + } const fingerprinted = result.translatedSnippets.map((snippet) => snippet.withFingerprint(this.fingerprinter.fingerprintAll(snippet.fqnsReferenced())), ); - if (addToTablet) { + if (options?.addToTablet ?? true) { for (const translation of fingerprinted) { this.tablet.addSnippet(translation); } @@ -261,3 +298,15 @@ export interface ReadFromCacheResults { readonly dirtyTypesCount: number; readonly dirtyDidntCompile: number; } + +export interface TranslateAllOptions { + /** + * @default - Create a temporary directory with all necessary packages + */ + readonly compilationDirectory?: string; + + /** + * @default true + */ + readonly addToTablet?: boolean; +} diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts new file mode 100644 index 0000000000..a21ace6710 --- /dev/null +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -0,0 +1,246 @@ +import * as cp from 'child_process'; +import * as fastGlob from 'fast-glob'; +import * as fs from 'fs-extra'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { intersect } from 'semver-intersect'; + +import { findDependencyDirectory, findUp } from './find-utils'; +import * as logging from './logging'; +import { TypeScriptSnippet, CompilationDependency } from './snippet'; +import { mkDict, formatList } from './util'; + +/** + * Collect the dependencies of a bunch of snippets together in one declaration + * + * We assume here the dependencies will not conflict. + */ +export function collectDependencies(snippets: TypeScriptSnippet[]) { + const ret: Record = {}; + for (const snippet of snippets) { + for (const [name, source] of Object.entries(snippet.compilationDependencies ?? {})) { + ret[name] = resolveConflict(name, source, ret[name]); + } + } + return ret; +} + +function resolveConflict( + name: string, + a: CompilationDependency, + b: CompilationDependency | undefined, +): CompilationDependency { + if (!b) { + return a; + } + + if (a.type === 'concrete' && b.type === 'concrete') { + if (b.resolvedDirectory !== a.resolvedDirectory) { + throw new Error(`Dependency conflict: ${name} can be either ${a.resolvedDirectory} or ${b.resolvedDirectory}`); + } + return a; + } + + if (a.type === 'symbolic' && b.type === 'symbolic') { + // Intersect the ranges + return { + type: 'symbolic', + versionRange: intersect(a.versionRange, b.versionRange), + }; + } + + if (a.type === 'concrete' && b.type === 'symbolic') { + const concreteVersion: string = fs.readJsonSync(path.join(a.resolvedDirectory, 'package.json')).version; + + if (!semver.satisfies(concreteVersion, b.versionRange)) { + throw new Error( + `Dependency conflict: ${name} expected to match ${b.versionRange} but found ${concreteVersion} at ${a.resolvedDirectory}`, + ); + } + + return a; + } + + if (a.type === 'symbolic' && b.type === 'concrete') { + // Reverse roles so we fall into the previous case + return resolveConflict(name, b, a); + } + + throw new Error('Cases should have been exhaustive'); +} + +/** + * Check that the directory we were given has all the necessary dependencies in it + * + * It's a warning if this is not true, not an error. + */ +export async function validateAvailableDependencies(directory: string, deps: Record) { + const failures = await Promise.all( + Object.entries(deps).flatMap(async ([name, _dep]) => { + try { + await findDependencyDirectory(name, directory); + return []; + } catch { + return [name]; + } + }), + ); + + if (failures.length > 0) { + logging.warn( + `${directory}: packages necessary to compile examples missing from supplied directory: ${failures.join(', ')}`, + ); + } +} + +/** + * Prepare a temporary directory with symlinks to all the dependencies we need. + * + * - Symlinks the concrete dependencies + * - Tries to first find the symbolic dependencies in a potential monorepo that might be present + * (try both `lerna` and `yarn` monorepos). + * - Installs the remaining symbolic dependencies using 'npm'. + */ +export async function prepareDependencyDirectory(deps: Record): Promise { + const concreteDirs = Object.values(deps) + .filter(isConcrete) + .map((x) => x.resolvedDirectory); + const monorepoPackages = await scanMonoRepos(concreteDirs); + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'rosetta')); + logging.info(`Preparing dependency closure at ${tmpDir}`); + + // Resolved symbolic packages against monorepo + const resolvedDeps = mkDict( + Object.entries(deps).map(([name, dep]) => [ + name, + dep.type === 'concrete' + ? dep + : ((monorepoPackages[name] + ? { type: 'concrete', resolvedDirectory: monorepoPackages[name] } + : dep) as CompilationDependency), + ]), + ); + + // Use 'npm install' only for the symbolic packages. For the concrete packages, + // npm is going to try and find transitive dependencies as well and it won't know + // about monorepos. + const symbolicInstalls = Object.entries(resolvedDeps).flatMap(([name, dep]) => + isSymbolic(dep) ? [`${name}@${dep.versionRange}`] : [], + ); + const linkedInstalls = mkDict( + Object.entries(resolvedDeps).flatMap(([name, dep]) => + isConcrete(dep) ? [[name, dep.resolvedDirectory] as const] : [], + ), + ); + + // Run 'npm install' on it + if (symbolicInstalls.length > 0) { + logging.debug(`Installing example dependencies: ${symbolicInstalls.join(' ')}`); + cp.execSync(`npm install ${symbolicInstalls.join(' ')}`, { cwd: tmpDir, encoding: 'utf-8' }); + } + + // Symlink the rest + if (Object.keys(linkedInstalls).length > 0) { + logging.debug(`Symlinking example dependencies: ${Object.values(linkedInstalls).join(' ')}`); + const modDir = path.join(tmpDir, 'node_modules'); + await Promise.all( + Object.entries(linkedInstalls).map(async ([name, source]) => { + const target = path.join(modDir, name); + if (!(await fs.pathExists(target))) { + // Package could be namespaced, so ensure the namespace dir exists + await fs.mkdirp(path.dirname(target)); + await fs.symlink(source, target, 'dir'); + } + }), + ); + } + + return tmpDir; +} + +/** + * Map package name to directory + */ +async function scanMonoRepos(startingDirs: readonly string[]): Promise> { + const globs = new Set(); + for (const dir of startingDirs) { + // eslint-disable-next-line no-await-in-loop + setExtend(globs, await findMonoRepoGlobs(dir)); + } + + if (globs.size === 0) { + return {}; + } + + logging.debug(`Monorepo package sources: ${Array.from(globs).join(', ')}`); + + const packageDirectories = await fastGlob(Array.from(globs).map(windowsToUnix), { onlyDirectories: true }); + const results = mkDict( + ( + await Promise.all( + packageDirectories.map(async (directory) => { + const pjLocation = path.join(directory, 'package.json'); + return (await fs.pathExists(pjLocation)) + ? [[(await fs.readJson(pjLocation)).name as string, directory] as const] + : []; + }), + ) + ).flat(), + ); + + logging.debug(`Found ${Object.keys(results).length} packages in monorepo: ${formatList(Object.keys(results))}`); + return results; +} + +async function findMonoRepoGlobs(startingDir: string): Promise> { + const ret = new Set(); + + // Lerna monorepo + const lernaJsonDir = await findUp(startingDir, async (dir) => fs.pathExists(path.join(dir, 'lerna.json'))); + if (lernaJsonDir) { + const lernaJson = await fs.readJson(path.join(lernaJsonDir, 'lerna.json')); + for (const glob of lernaJson?.packages ?? []) { + ret.add(path.join(lernaJsonDir, glob)); + } + } + + // Yarn monorepo + const yarnWsDir = await findUp( + startingDir, + async (dir) => + (await fs.pathExists(path.join(dir, 'package.json'))) && + (await fs.readJson(path.join(dir, 'package.json')))?.workspaces !== undefined, + ); + if (yarnWsDir) { + const yarnWs = await fs.readJson(path.join(yarnWsDir, 'package.json')); + for (const glob of yarnWs.workspaces?.packages ?? []) { + ret.add(path.join(yarnWsDir, glob)); + } + } + + return ret; +} + +function isSymbolic(x: CompilationDependency): x is Extract { + return x.type === 'symbolic'; +} + +function isConcrete(x: CompilationDependency): x is Extract { + return x.type === 'concrete'; +} + +function setExtend(xs: Set, ys: Set) { + for (const y of ys) { + xs.add(y); + } + return xs; +} + +/** + * Necessary for fastGlob + */ +function windowsToUnix(x: string) { + return x.replace(/\\/g, '/'); +} diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts index 0b4348ba9b..33a7753ac4 100644 --- a/packages/jsii-rosetta/lib/snippet.ts +++ b/packages/jsii-rosetta/lib/snippet.ts @@ -30,8 +30,22 @@ export interface TypeScriptSnippet { * @default false */ readonly strict?: boolean; + + /** + * Dependencies necessary to compile this snippet + * + * Value is a regular { name -> semver } map like NPM's `dependencies`, + * `devDependencies` etc. + * + * @default none + */ + readonly compilationDependencies?: Record; } +export type CompilationDependency = + | { readonly type: 'concrete'; readonly resolvedDirectory: string } + | { readonly type: 'symbolic'; readonly versionRange: string }; + /** * Description of a location where the snippet is found * @@ -279,6 +293,20 @@ export enum SnippetParameters { */ LITERATE_SOURCE = 'lit', + /** + * This snippet has been infused + * + * This means it has been copied from a different location, and potentially + * even from a different assembly. If so, we can't expect it to compile in + * the future, and if doesn't, we ignore the errors. + * + * N.B: this shouldn't make a difference in normal operation, as the `infuse` + * command will duplicate the translation to the target tablet. This only + * matters if we remove the tablet and try to re-extract an assembly with + * infused examples from somewher else. + */ + INFUSED = 'infused', + /** * What directory to resolve fixtures in for this snippet (system parameter) * diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index 85c515ffa4..40fd85f0d4 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { TargetLanguage } from '../languages'; import * as logging from '../logging'; import { TypeScriptSnippet, SnippetLocation, completeSource } from '../snippet'; -import { mapValues } from '../util'; +import { mapValues, Mutable } from '../util'; import { snippetKey } from './key'; import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './schema'; @@ -282,5 +282,3 @@ export interface Translation { language: string; didCompile?: boolean; } - -type Mutable = { -readonly [P in keyof T]: Mutable }; diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts index 93ba9d7fd6..d35dd1d375 100644 --- a/packages/jsii-rosetta/lib/util.ts +++ b/packages/jsii-rosetta/lib/util.ts @@ -24,6 +24,12 @@ export function printDiagnostics(diags: readonly RosettaDiagnostic[], stream: No } } +export function formatList(xs: string[], n = 5) { + const tooMany = xs.length - n; + + return tooMany > 0 ? `${xs.slice(0, n).join(', ')} (and ${tooMany} more)` : xs.join(', '); +} + export const StrictBrand = 'jsii.strict'; interface MaybeStrictDiagnostic { readonly [StrictBrand]?: boolean; @@ -206,6 +212,8 @@ export function indexBy(xs: A[], fn: (x: A) => string): Record { return mkDict(xs.map((x) => [fn(x), x] as const)); } +export type Mutable = { -readonly [P in keyof T]: Mutable }; + export function commentToken(language: string) { // This is future-proofed a bit, but don't read too much in this... switch (language) { diff --git a/packages/jsii-rosetta/package.json b/packages/jsii-rosetta/package.json index 64ac75f641..d1d36fb666 100644 --- a/packages/jsii-rosetta/package.json +++ b/packages/jsii-rosetta/package.json @@ -22,6 +22,7 @@ "@types/mock-fs": "^4.13.1", "@types/node": "^12.20.43", "@types/workerpool": "^6.1.0", + "@types/semver": "^7.3.9", "eslint": "^8.8.0", "jest": "^27.4.7", "jsii-build-tools": "0.0.0", @@ -39,6 +40,9 @@ "@xmldom/xmldom": "^0.8.0", "workerpool": "^6.2.0", "yargs": "^16.2.0", + "semver": "^7.3.5", + "semver-intersect": "^1.4.0", + "fast-glob": "^3.2.7", "jsii": "0.0.0" }, "license": "Apache-2.0", diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index 51406b4629..1fae776890 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -1,4 +1,5 @@ import * as fs from 'fs-extra'; +import { compileJsiiForTest } from 'jsii'; import * as path from 'path'; import { @@ -12,8 +13,11 @@ import { import * as extract from '../../lib/commands/extract'; import { loadAssemblies } from '../../lib/jsii/assemblies'; import { TARGET_LANGUAGES } from '../../lib/languages'; +import * as logging from '../../lib/logging'; import { TestJsiiModule, DUMMY_JSII_CONFIG, testSnippetLocation } from '../testutil'; +jest.setTimeout(30_000); + const DUMMY_README = ` Here is an example of how to use ClassA: @@ -440,8 +444,8 @@ describe('infused examples', () => { 'index.ts': ` /** * ClassA - * - * @exampleMetadata infused + * + * @exampleMetadata infused * @example x */ export class ClassA { @@ -520,13 +524,106 @@ describe('infused examples', () => { }); }); +test('can use additional dependencies from monorepo', async () => { + logging.configure({ level: logging.Level.VERBOSE }); + const asm = await TestJsiiModule.fromSource( + { + 'index.ts': ` + /** + * Class to hold values + * + * @example + * import { ValueHolder } from 'my_assembly'; + * import { SomeClass } from 'otherModule'; + * new ValueHolder(new SomeClass()); + */ + export class ValueHolder { + constructor(public readonly theValue: any) { } + } + `, + }, + { + name: 'my_assembly', + jsii: DUMMY_JSII_CONFIG, + jsiiRosetta: { + exampleDependencies: { + // This relies on the fact that Rosetta will find the package in the monorepo + otherModule: '*', + }, + }, + }, + ); + try { + // GIVEN - install some random other module + await asm.workspace.addDependency( + await compileJsiiForTest( + { + 'index.ts': 'export class SomeClass { }', + }, + { + packageJson: { + name: 'otherModule', + }, + }, + ), + ); + // GIVEN - a lerna.json that would find that package + await fs.writeJson(path.join(asm.workspaceDirectory, 'lerna.json'), { + packages: ['node_modules/*'], + }); + + // WHEN + await extract.extractSnippets([asm.moduleDirectory], defaultExtractOptions); + // THEN -- did not throw an error + } finally { + await asm.cleanup(); + } +}); + +test('can use additional dependencies from NPM', async () => { + const asm = await TestJsiiModule.fromSource( + { + 'index.ts': ` + /** + * Class to hold values + * + * @example + * import { ValueHolder } from 'my_assembly'; + * import { ConstructOrder } from 'constructs'; + * new ValueHolder(ConstructOrder.PREORDER); + */ + export class ValueHolder { + constructor(public readonly theValue: any) { } + } + `, + }, + { + name: 'my_assembly', + jsii: DUMMY_JSII_CONFIG, + jsiiRosetta: { + exampleDependencies: { + // This relies on the fact that Rosetta will find the package in the monorepo + constructs: '^10.0.0', + }, + }, + }, + ); + try { + // WHEN + await extract.extractSnippets([asm.moduleDirectory], defaultExtractOptions); + // THEN -- did not throw an error + } finally { + await asm.cleanup(); + } +}); + test('infused examples have no diagnostics', async () => { const otherAssembly = await TestJsiiModule.fromSource( { 'index.ts': ` /** * ClassA - * + * * @exampleMetadata infused * @example x */ diff --git a/packages/jsii-rosetta/test/commands/infuse.test.ts b/packages/jsii-rosetta/test/commands/infuse.test.ts index 8962d0522f..f8e8b7c316 100644 --- a/packages/jsii-rosetta/test/commands/infuse.test.ts +++ b/packages/jsii-rosetta/test/commands/infuse.test.ts @@ -8,6 +8,8 @@ import { infuse, DEFAULT_INFUSION_RESULTS_NAME } from '../../lib/commands/infuse import { loadAssemblies } from '../../lib/jsii/assemblies'; import { TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil'; +jest.setTimeout(30_000); + const DUMMY_README = ` Here is an example of how to use ClassA: diff --git a/packages/jsii-rosetta/test/jsii/assemblies.test.ts b/packages/jsii-rosetta/test/jsii/assemblies.test.ts index 97878d9a9f..a5cf9d0d7a 100644 --- a/packages/jsii-rosetta/test/jsii/assemblies.test.ts +++ b/packages/jsii-rosetta/test/jsii/assemblies.test.ts @@ -8,9 +8,9 @@ import { SnippetParameters } from '../../lib/snippet'; import { TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil'; import { fakeAssembly } from './fake-assembly'; -test('Extract snippet from README', () => { +test('Extract snippet from README', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -25,9 +25,9 @@ test('Extract snippet from README', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Extract snippet from submodule READMEs', () => { +test('Extract snippet from submodule READMEs', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ submodules: { @@ -46,9 +46,9 @@ test('Extract snippet from submodule READMEs', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Extract snippet from type docstring', () => { +test('Extract snippet from type docstring', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -72,9 +72,9 @@ test('Extract snippet from type docstring', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Snippet can include fixture', () => { +test('Snippet can include fixture', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -109,9 +109,9 @@ test('Snippet can include fixture', () => { `); }); -test('Use fixture from example', () => { +test('Use fixture from example', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -148,9 +148,9 @@ test('Use fixture from example', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Fixture allows use of import statements', () => { +test('Fixture allows use of import statements', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -198,14 +198,14 @@ test('Fixture allows use of import statements', () => { ); }); -test('Backwards compatibility with literate integ tests', () => { +test('Backwards compatibility with literate integ tests', async () => { mockfs({ '/package/test/integ.example.lit.ts': '# Some literate source file', }); try { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -262,7 +262,9 @@ test('rosetta fixture from submodule is preferred if it exists', async () => { 'dont pick me\n/// here', ); - const snippets = allTypeScriptSnippets([{ assembly: jsiiModule.assembly, directory: jsiiModule.moduleDirectory }]); + const snippets = await allTypeScriptSnippets([ + { assembly: jsiiModule.assembly, directory: jsiiModule.moduleDirectory }, + ]); expect(snippets[0].completeSource).toMatch(/^pick me/); } finally { diff --git a/packages/jsii-rosetta/test/testutil.ts b/packages/jsii-rosetta/test/testutil.ts index 953ae3c5fa..1c3011da63 100644 --- a/packages/jsii-rosetta/test/testutil.ts +++ b/packages/jsii-rosetta/test/testutil.ts @@ -35,7 +35,7 @@ export class TestJsiiModule { public readonly moduleDirectory: string; public readonly workspaceDirectory: string; - private constructor(public readonly assembly: spec.Assembly, private readonly workspace: TestWorkspace) { + private constructor(public readonly assembly: spec.Assembly, public readonly workspace: TestWorkspace) { this.moduleDirectory = workspace.dependencyDir(assembly.name); this.workspaceDirectory = workspace.rootDirectory; } diff --git a/packages/jsii-rosetta/vendor/semver-intersect.d.ts b/packages/jsii-rosetta/vendor/semver-intersect.d.ts new file mode 100644 index 0000000000..c7f919bcf7 --- /dev/null +++ b/packages/jsii-rosetta/vendor/semver-intersect.d.ts @@ -0,0 +1,13 @@ +/// Hand-written declaration for the semver-intersect module +declare module 'semver-intersect' { + /** + * Computes the intersection between multiple semver ranges. + * + * @param ranges the ranges for which the intersection is requested. + * + * @returns the intersection of `ranges`. + * + * @throws Error if the intersection is empty. + */ + function intersect(...ranges: string[]): string; +} diff --git a/packages/jsii/lib/helpers.ts b/packages/jsii/lib/helpers.ts index 37e09040b4..5f5bd579cc 100644 --- a/packages/jsii/lib/helpers.ts +++ b/packages/jsii/lib/helpers.ts @@ -217,11 +217,15 @@ export interface TestCompilationOptions { /** * Parts of projectInfo to override (package name etc) + * + * @deprecated Prefer using `packageJson` instead. */ readonly projectInfo?: Partial; /** * Parts of projectInfo to override (package name etc) + * + * @default - Use some default values */ readonly packageJson?: Partial; } diff --git a/yarn.lock b/yarn.lock index 39be25c52c..b833de421f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3572,6 +3572,17 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== +fast-glob@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" + integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + fast-glob@^3.2.9: version "3.2.11" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.11.tgz#a1172ad95ceb8a16e20caa5c5e56480e5129c1d9"