From ed33b57f7ca7fbc184a519b2b3de717b8a6eea88 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 16 Dec 2021 15:52:34 +0100 Subject: [PATCH 01/16] feat(rosetta): Rosetta manages dependencies automatically --- packages/@jsii/spec/lib/configuration.ts | 2 + packages/jsii-rosetta/README.md | 38 +++- packages/jsii-rosetta/bin/jsii-rosetta.ts | 9 +- packages/jsii-rosetta/lib/commands/extract.ts | 10 +- packages/jsii-rosetta/lib/commands/infuse.ts | 6 +- .../lib/commands/transliterate.ts | 3 +- .../jsii-rosetta/lib/commands/trim-cache.ts | 2 +- packages/jsii-rosetta/lib/find-utils.ts | 86 ++++++++ packages/jsii-rosetta/lib/jsii/assemblies.ts | 165 +++++++++++--- packages/jsii-rosetta/lib/jsii/packages.ts | 41 ---- packages/jsii-rosetta/lib/rosetta-reader.ts | 2 +- .../jsii-rosetta/lib/rosetta-translator.ts | 52 ++++- .../jsii-rosetta/lib/snippet-dependencies.ts | 201 ++++++++++++++++++ packages/jsii-rosetta/lib/snippet.ts | 28 +++ packages/jsii-rosetta/lib/tablets/tablets.ts | 4 +- packages/jsii-rosetta/lib/util.ts | 2 + packages/jsii-rosetta/package.json | 4 + .../jsii-rosetta/test/jsii/assemblies.test.ts | 32 +-- yarn.lock | 2 +- 19 files changed, 567 insertions(+), 122 deletions(-) create mode 100644 packages/jsii-rosetta/lib/find-utils.ts create mode 100644 packages/jsii-rosetta/lib/snippet-dependencies.ts diff --git a/packages/@jsii/spec/lib/configuration.ts b/packages/@jsii/spec/lib/configuration.ts index ca5e9fffec..57f13f5c3e 100644 --- a/packages/@jsii/spec/lib/configuration.ts +++ b/packages/@jsii/spec/lib/configuration.ts @@ -171,5 +171,7 @@ export interface PackageJson { */ devDependencies?: Record; + bundledDependencies?: string[]; + [key: string]: unknown; } diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md index 7b1e32c7a8..6139d4b9cb 100644 --- a/packages/jsii-rosetta/README.md +++ b/packages/jsii-rosetta/README.md @@ -44,20 +44,14 @@ someObject.someMethod('foo', { ### Enforcing correct examples By default, Rosetta will accept non-compiling examples. If you set -`jsii.metadata.jsii.rosetta.strict` to `true` in your `package.json`, +`jsiiRosetta.strict` to `true` in your `package.json`, the Rosetta command will fail if any example contains an error: ```js /// package.json { - "jsii": { - "metadata": { - "jsii": { - "rosetta": { - "strict": true - } - } - } + "jsiiRosetta": { + "strict": true } } ``` @@ -114,6 +108,32 @@ To specify fixtures in an `@example` block, use an accompanying `@exampleMetadat */ ```` +### Dependencies + +When compiling examples, Rosetta will make sure your package itself and all of +its `dependencies` and `peerDependencies` are available in the dependency +closure that your examples will be compiled in. + +If there are packages you want to use in an example that should *not* be part +of your package's dependencies, declare them in `jsiiRosetta.exampleDependencies` +in your `package.json`: + +```js +/// package.json +{ + "jsiiRosetta": { + "exampleDependencies": { + "@some-other/package": "^1.2.3", + "@yet-another/package": "*", + } + } +} +``` + +You can also set up a directory with correct dependencies yourself, and pass +`--directory` when running `jsii-rosetta extract`. We recommend using the +automatic closure building mechanism and specifying `exampleDependencies` though. + ## Rosetta for package publishers This section describes how Rosetta integrates into your build process. diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts index a3e059f18b..c3e55ca12a 100644 --- a/packages/jsii-rosetta/bin/jsii-rosetta.ts +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -210,20 +210,13 @@ function main() { args.fail = args.f = true; } - // Easiest way to get a fixed working directory (for sources) in is to - // chdir, since underneath the in-memory layer we're using a regular TS - // compilerhost. Have to make all file references absolute before we chdir - // though. const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x)); const absCacheFrom = fmap(args.cache ?? args['cache-from'], path.resolve); const absCacheTo = fmap(args.cache ?? args['cache-to'] ?? args.output, path.resolve); - if (args.directory) { - process.chdir(args.directory); - } - const extractOptions: ExtractOptions = { + compilationDirectory: args.directory, includeCompilerDiagnostics: !!args.compile, validateAssemblies: args['validate-assemblies'], only: args.include, diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index 052c187668..9ff1b5d216 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -37,6 +37,14 @@ export interface ExtractOptions { */ readonly trimCache?: boolean; + /** + * What directory to compile the samples in + * + * @default - Rosetta manages the compilation directory + * @deprecated Samples declare their own dependencies instead + */ + readonly compilationDirectory?: string; + /** * Make a translator (just for testing) */ @@ -69,7 +77,7 @@ export async function extractSnippets( logging.info(`Loading ${assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies ?? false); - let snippets = Array.from(allTypeScriptSnippets(assemblies, loose)); + let snippets = Array.from(await allTypeScriptSnippets(assemblies, loose)); if (only.length > 0) { snippets = filterSnippets(snippets, only); } diff --git a/packages/jsii-rosetta/lib/commands/infuse.ts b/packages/jsii-rosetta/lib/commands/infuse.ts index 4820309c9e..183977b3d8 100644 --- a/packages/jsii-rosetta/lib/commands/infuse.ts +++ b/packages/jsii-rosetta/lib/commands/infuse.ts @@ -75,7 +75,7 @@ export async function infuse(assemblyLocations: string[], options?: InfuseOption } availableTranslations.addTablets(...Object.values(defaultTablets)); - const { translationsByFqn, originalsByKey } = availableSnippetsPerFqn(assemblies, availableTranslations); + const { translationsByFqn, originalsByKey } = await availableSnippetsPerFqn(assemblies, availableTranslations); const additionalOutputTablet = options?.cacheToFile ? await LanguageTablet.fromOptionalFile(options?.cacheToFile) @@ -244,10 +244,10 @@ function insertExample( * * Returns a map of fqns to a list of keys that represent snippets that include the fqn. */ -function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) { +async function availableSnippetsPerFqn(asms: readonly LoadedAssembly[], translationsTablet: LanguageTablet) { const ret = new DefaultRecord(); - const originalsByKey = indexBy(allTypeScriptSnippets(asms), snippetKey); + const originalsByKey = indexBy(await allTypeScriptSnippets(asms), snippetKey); const translations = Object.keys(originalsByKey) .map((key) => translationsTablet.tryGetSnippet(key)) diff --git a/packages/jsii-rosetta/lib/commands/transliterate.ts b/packages/jsii-rosetta/lib/commands/transliterate.ts index 2f00bdd7ca..6be40cc24f 100644 --- a/packages/jsii-rosetta/lib/commands/transliterate.ts +++ b/packages/jsii-rosetta/lib/commands/transliterate.ts @@ -9,7 +9,7 @@ import { debug } from '../logging'; import { RosettaTabletReader, UnknownSnippetMode } from '../rosetta-reader'; import { SnippetParameters, typeScriptSnippetFromVisibleSource, ApiLocation, parseMetadataLine } from '../snippet'; import { Translation } from '../tablets/tablets'; -import { fmap } from '../util'; +import { fmap, Mutable } from '../util'; export interface TransliterateAssemblyOptions { /** @@ -126,7 +126,6 @@ async function loadAssemblies( return result; } -type Mutable = { -readonly [K in keyof T]: Mutable }; type AssemblyLoader = () => Promise>; function prefixDisclaimer(translation: Translation): string { diff --git a/packages/jsii-rosetta/lib/commands/trim-cache.ts b/packages/jsii-rosetta/lib/commands/trim-cache.ts index f22ce4aa81..de87328489 100644 --- a/packages/jsii-rosetta/lib/commands/trim-cache.ts +++ b/packages/jsii-rosetta/lib/commands/trim-cache.ts @@ -20,7 +20,7 @@ export async function trimCache(options: TrimCacheOptions): Promise { logging.info(`Loading ${options.assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(options.assemblyLocations, false); - const snippets = Array.from(allTypeScriptSnippets(assemblies)); + const snippets = Array.from(await allTypeScriptSnippets(assemblies)); const original = await LanguageTablet.fromFile(options.cacheFile); const updated = new LanguageTablet(); diff --git a/packages/jsii-rosetta/lib/find-utils.ts b/packages/jsii-rosetta/lib/find-utils.ts new file mode 100644 index 0000000000..807aa334f5 --- /dev/null +++ b/packages/jsii-rosetta/lib/find-utils.ts @@ -0,0 +1,86 @@ +import * as fs from 'fs-extra'; +import * as path from 'path'; + +/** + * Find the directory that contains a given dependency, identified by its 'package.json', from a starting search directory + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export async function findDependencyDirectory(dependencyName: string, searchStart: string) { + // Explicitly do not use 'require("dep/package.json")' because that will fail if the + // package does not export that particular file. + const entryPoint = require.resolve(dependencyName, { + paths: [searchStart], + }); + + // Search up from the given directory, looking for a package.json that matches + // the dependency name (so we don't accidentally find stray 'package.jsons'). + const depPkgJsonPath = await findPackageJsonUp(dependencyName, path.dirname(entryPoint)); + + if (!depPkgJsonPath) { + throw new Error(`Could not find dependency '${dependencyName}' from '${searchStart}'`); + } + + return depPkgJsonPath; +} + +/** + * Find the package.json for a given package upwards from the given directory + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export async function findPackageJsonUp(packageName: string, directory: string) { + return findUp(directory, async (dir) => { + const pjFile = path.join(dir, 'package.json'); + return (await fs.pathExists(pjFile)) && (await fs.readJson(pjFile)).name === packageName; + }); +} + +/** + * Find a directory up the tree from a starting directory matching a condition + * + * Will return `undefined` if no directory matches + * + * (This code is duplicated among jsii/jsii-pacmak/jsii-reflect. Changes should be done in all + * 3 locations, and we should unify these at some point: https://github.com/aws/jsii/issues/3236) + */ +export function findUp(directory: string, pred: (dir: string) => Promise): Promise; +export function findUp(directory: string, pred: (dir: string) => boolean): string | undefined; +// eslint-disable-next-line @typescript-eslint/promise-function-async +export function findUp( + directory: string, + pred: ((dir: string) => boolean) | ((dir: string) => Promise), +): Promise | string | undefined { + const result = pred(directory); + if (isPromise(result)) { + return result.then((thisDirectory) => (thisDirectory ? directory : recurse())); + } + + return result ? directory : recurse(); + + function recurse() { + const parent = path.dirname(directory); + if (parent === directory) { + return undefined; + } + return findUp(parent, pred as any); + } +} + +function isPromise(x: A | Promise): x is Promise { + return typeof x === 'object' && (x as any).then; +} + +/** + * Whether the given dependency is a built-in + * + * Some dependencies that occur in `package.json` are also built-ins in modern Node + * versions (most egregious example: 'punycode'). Detect those and filter them out. + */ +export function isBuiltinModule(depName: string) { + // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires + const { builtinModules } = require('module'); + return (builtinModules ?? []).includes(depName); +} diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts index 0e1463d71d..baeed38162 100644 --- a/packages/jsii-rosetta/lib/jsii/assemblies.ts +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -3,6 +3,7 @@ import * as crypto from 'crypto'; import * as fs from 'fs-extra'; import * as path from 'path'; +import { findDependencyDirectory, isBuiltinModule } from '../find-utils'; import { fixturize } from '../fixtures'; import { extractTypescriptSnippetsFromMarkdown } from '../markdown/extract-snippets'; import { @@ -12,6 +13,7 @@ import { SnippetParameters, ApiLocation, parseMetadataLine, + CompilationDependency, } from '../snippet'; import { enforcesStrictMode } from '../strict'; import { LanguageTablet, DEFAULT_TABLET_NAME } from '../tablets/tablets'; @@ -36,9 +38,17 @@ const sortJson = require('sort-json'); */ export const EXAMPLE_METADATA_JSDOCTAG = 'exampleMetadata'; +interface RosettaPackageJson extends spec.PackageJson { + readonly jsiiRosetta?: { + readonly strict?: boolean; + readonly exampleDependencies?: Record; + }; +} + export interface LoadedAssembly { readonly assembly: spec.Assembly; readonly directory: string; + readonly packageJson?: RosettaPackageJson; } /** @@ -56,10 +66,15 @@ export async function loadAssemblies( return loadAssembly(path.join(location, '.jsii')); } - return { - assembly: await loadAssemblyFromFile(location, validateAssemblies), - directory: path.dirname(location), - }; + const directory = path.dirname(location); + const pjLocation = path.join(directory, 'package.json'); + + const [assembly, packageJson] = await Promise.all([ + loadAssemblyFromFile(location, validateAssemblies), + (await fs.pathExists(pjLocation)) ? fs.readJSON(pjLocation, { encoding: 'utf-8' }) : Promise.resolve(undefined), + ]); + + return { assembly, directory, packageJson }; } } @@ -151,36 +166,50 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc } } -export function allTypeScriptSnippets(assemblies: readonly LoadedAssembly[], loose = false): TypeScriptSnippet[] { - const ret = new Array(); - - for (const { assembly, directory } of assemblies) { - const strict = enforcesStrictMode(assembly); - for (const source of allSnippetSources(assembly)) { - switch (source.type) { - case 'example': - // If an example is an infused example, we do not care about compiler errors. - // We are relying on the tablet cache to have this example stored already. - const [strictForExample, looseForExample] = - source.metadata?.infused !== undefined ? [false, true] : [strict, loose]; - const location = { api: source.location, field: { field: 'example' } } as const; - const snippet = updateParameters(typeScriptSnippetFromSource(source.source, location, strictForExample), { - [SnippetParameters.$PROJECT_DIRECTORY]: directory, - ...source.metadata, - }); - ret.push(fixturize(snippet, looseForExample)); - break; - case 'markdown': - for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.location, strict)) { - const withDirectory = updateParameters(snippet, { - [SnippetParameters.$PROJECT_DIRECTORY]: directory, - }); - ret.push(fixturize(withDirectory, loose)); - } - } - } - } - return ret; +export async function allTypeScriptSnippets( + assemblies: readonly LoadedAssembly[], + loose = false, +): Promise { + return Promise.all( + assemblies + .flatMap((loaded) => allSnippetSources(loaded.assembly).map((source) => ({ source, loaded }))) + .flatMap(({ source, loaded }) => { + switch (source.type) { + case 'example': + return [ + { + snippet: updateParameters( + typeScriptSnippetFromSource( + source.source, + { api: source.location, field: { field: 'example' } }, + isStrict(loaded), + ), + source.metadata ?? {}, + ), + loaded, + }, + ]; + case 'markdown': + return extractTypescriptSnippetsFromMarkdown(source.markdown, source.location, isStrict(loaded)).map( + (snippet) => ({ snippet, loaded }), + ); + } + }) + .map(async ({ snippet, loaded }) => { + const isInfused = snippet.parameters?.infused !== undefined; + + // Ignore fixturization errors if requested on this command, or if the snippet was infused + const ignoreFixtureErrors = loose || isInfused; + + // Also if the snippet was infused: switch off 'strict' mode if it was set + if (isInfused) { + snippet = { ...snippet, strict: false }; + } + + snippet = await withDependencies(loaded, withProjectDirectory(loaded.directory, snippet)); + return fixturize(snippet, ignoreFixtureErrors); + }), + ); } /** @@ -298,3 +327,71 @@ export function findContainingSubmodule(assembly: spec.Assembly, fqn: string): s } return undefined; } + +function withProjectDirectory(dir: string, snippet: TypeScriptSnippet) { + return updateParameters(snippet, { + [SnippetParameters.$PROJECT_DIRECTORY]: dir, + }); +} + +/** + * Return a TypeScript snippet with dependencies added + * + * The dependencies will be taken from the package.json, and will consist of: + * + * - The package itself + * - The package's dependencies and peerDependencies + * - Any additional dependencies declared in `jsiiRosetta.exampleDependencies`. + */ +async function withDependencies(asm: LoadedAssembly, snippet: TypeScriptSnippet): Promise { + const compilationDependencies: Record = {}; + + compilationDependencies[asm.assembly.name] = { + type: 'concrete', + resolvedDirectory: await fs.realpath(asm.directory), + }; + + Object.assign( + compilationDependencies, + mkDict( + await Promise.all( + Object.keys({ ...asm.packageJson?.dependencies, ...asm.packageJson?.peerDependencies }) + .filter((name) => !isBuiltinModule(name)) + .filter((name) => !(asm.packageJson?.bundledDependencies ?? []).includes(name)) + .map( + async (name) => + [ + name, + { + type: 'concrete', + resolvedDirectory: await fs.realpath(await findDependencyDirectory(name, asm.directory)), + }, + ] as const, + ), + ), + ), + ); + + Object.assign( + compilationDependencies, + mkDict( + Object.entries(asm.packageJson?.jsiiRosetta?.exampleDependencies ?? {}).map( + ([name, versionRange]) => [name, { type: 'symbolic', versionRange }] as const, + ), + ), + ); + + return { + ...snippet, + compilationDependencies, + }; +} + +/** + * Whether samples in the assembly should be treated as strict + * + * True if the strict flag is found in the package.json (modern) or the assembly itself (legacy). + */ +function isStrict(loaded: LoadedAssembly) { + return loaded.packageJson?.jsiiRosetta?.strict ?? enforcesStrictMode(loaded.assembly); +} diff --git a/packages/jsii-rosetta/lib/jsii/packages.ts b/packages/jsii-rosetta/lib/jsii/packages.ts index 9aa5e2ec61..b1daae3fa6 100644 --- a/packages/jsii-rosetta/lib/jsii/packages.ts +++ b/packages/jsii-rosetta/lib/jsii/packages.ts @@ -1,45 +1,4 @@ import * as spec from '@jsii/spec'; -import * as fs from 'fs'; -import * as path from 'path'; - -/** - * Resolve a package name in an example to a JSII assembly - * - * We assume we've changed directory to the directory where we need to resolve from. - */ -export function resolvePackage(packageName: string) { - try { - const resolved = require.resolve(`${packageName}/package.json`, { - paths: [process.cwd()], - }); - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(resolved); - } catch { - return undefined; - } -} - -/** - * Find an enclosing package.json file given a filename - * - * Will return `undefined` if a package.json could not be found. - */ -export function findPackageJson(fileName: string) { - // eslint-disable-next-line no-constant-condition - while (true) { - const candidatePath = path.join(fileName, 'package.json'); - if (fs.existsSync(candidatePath)) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(path.resolve(candidatePath)); - } - - const parent = path.dirname(fileName); - if (parent === fileName) { - return undefined; - } - fileName = parent; - } -} export function jsiiTargetParameter(target: spec.Targetable, field: string) { const path = field.split('.'); diff --git a/packages/jsii-rosetta/lib/rosetta-reader.ts b/packages/jsii-rosetta/lib/rosetta-reader.ts index 4e96a5b674..aa61492d25 100644 --- a/packages/jsii-rosetta/lib/rosetta-reader.ts +++ b/packages/jsii-rosetta/lib/rosetta-reader.ts @@ -155,7 +155,7 @@ export class RosettaTabletReader { // Inventarize the snippets from this assembly, but only if there's a chance // we're going to need them. if (this.unknownSnippets === UnknownSnippetMode.TRANSLATE) { - for (const tsnip of allTypeScriptSnippets([{ assembly, directory: assemblyDir }], this.loose)) { + for (const tsnip of await allTypeScriptSnippets([{ assembly, directory: assemblyDir }], this.loose)) { this.extractedSnippets.set(snippetKey(tsnip), tsnip); } } diff --git a/packages/jsii-rosetta/lib/rosetta-translator.ts b/packages/jsii-rosetta/lib/rosetta-translator.ts index b5f8c7d0ee..ed17bc2fee 100644 --- a/packages/jsii-rosetta/lib/rosetta-translator.ts +++ b/packages/jsii-rosetta/lib/rosetta-translator.ts @@ -1,9 +1,11 @@ import * as spec from '@jsii/spec'; +import * as fs from 'fs-extra'; import { TypeFingerprinter } from './jsii/fingerprinting'; import { TARGET_LANGUAGES } from './languages'; import * as logging from './logging'; import { TypeScriptSnippet, completeSource } from './snippet'; +import { collectDependencies, validateAvailableDependencies, prepareDependencyDirectory } from './snippet-dependencies'; import { snippetKey } from './tablets/key'; import { LanguageTablet, TranslatedSnippet } from './tablets/tablets'; import { translateAll, TranslateAllResult } from './translate_all'; @@ -105,14 +107,49 @@ export class RosettaTranslator { return { translations, remaining }; } - public async translateAll(snippets: TypeScriptSnippet[], addToTablet = true): Promise { - const result = await translateAll(snippets, this.includeCompilerDiagnostics); + public async translateAll(snippets: TypeScriptSnippet[], addToTablet?: boolean): Promise; + public async translateAll(snippets: TypeScriptSnippet[], options?: TranslateAllOptions): Promise; + public async translateAll( + snippets: TypeScriptSnippet[], + optionsOrAddToTablet?: boolean | TranslateAllOptions, + ): Promise { + const options = + optionsOrAddToTablet && typeof optionsOrAddToTablet === 'object' + ? optionsOrAddToTablet + : { addToTablet: optionsOrAddToTablet }; + + const exampleDependencies = collectDependencies(snippets); + + let compilationDirectory; + let cleanCompilationDir = false; + if (options?.compilationDirectory) { + // If the user provided a directory, we're going to trust-but-confirm. + await validateAvailableDependencies(options.compilationDirectory, exampleDependencies); + compilationDirectory = options.compilationDirectory; + } else { + compilationDirectory = await prepareDependencyDirectory(exampleDependencies); + cleanCompilationDir = true; + } + + const origDir = process.cwd(); + // Easiest way to get a fixed working directory (for sources) in is to chdir + process.chdir(compilationDirectory); + + let result; + try { + result = await translateAll(snippets, this.includeCompilerDiagnostics); + } finally { + process.chdir(origDir); + if (cleanCompilationDir) { + await fs.remove(compilationDirectory); + } + } const fingerprinted = result.translatedSnippets.map((snippet) => snippet.withFingerprint(this.fingerprinter.fingerprintAll(snippet.fqnsReferenced())), ); - if (addToTablet) { + if (options?.addToTablet ?? true) { for (const translation of fingerprinted) { this.tablet.addSnippet(translation); } @@ -158,3 +195,12 @@ export interface ReadFromCacheResults { readonly translations: TranslatedSnippet[]; readonly remaining: TypeScriptSnippet[]; } + +export interface TranslateAllOptions { + readonly compilationDirectory?: string; + + /** + * @default true + */ + readonly addToTablet?: boolean; +} diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts new file mode 100644 index 0000000000..c5b8054deb --- /dev/null +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -0,0 +1,201 @@ +import * as cp from 'child_process'; +import * as fastGlob from 'fast-glob'; +import * as fs from 'fs-extra'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { intersect } from 'semver-intersect'; + +import { findDependencyDirectory, findUp } from './find-utils'; +import * as logging from './logging'; +import { TypeScriptSnippet, CompilationDependency } from './snippet'; +import { mkDict } from './util'; + +/** + * Collect the dependencies of a bunch of snippets together in one declaration + * + * We assume here the dependencies will not conflict. + */ +export function collectDependencies(snippets: TypeScriptSnippet[]) { + const ret: Record = {}; + for (const snippet of snippets) { + for (const [name, source] of Object.entries(snippet.compilationDependencies ?? {})) { + ret[name] = resolveConflict(name, source, ret[name]); + } + } + return ret; +} + +function resolveConflict( + name: string, + a: CompilationDependency, + b: CompilationDependency | undefined, +): CompilationDependency { + if (!b) { + return a; + } + + if (a.type === 'concrete' && b.type === 'concrete') { + if (b.resolvedDirectory !== a.resolvedDirectory) { + throw new Error(`Dependency conflict: ${name} can be either ${a.resolvedDirectory} or ${b.resolvedDirectory}`); + } + return a; + } + + if (a.type === 'symbolic' && b.type === 'symbolic') { + // Intersect the ranges + return { + type: 'symbolic', + versionRange: intersect(a.versionRange, b.versionRange), + }; + } + + if (a.type === 'concrete' && b.type === 'symbolic') { + const concreteVersion: string = fs.readJsonSync(path.join(a.resolvedDirectory, 'package.json')).version; + + if (!semver.satisfies(concreteVersion, b.versionRange)) { + throw new Error( + `Dependency conflict: ${name} expected to match ${b.versionRange} but found ${concreteVersion} at ${a.resolvedDirectory}`, + ); + } + + return a; + } + + if (a.type === 'symbolic' && b.type === 'concrete') { + // Reverse roles so we fall into the previous case + return resolveConflict(name, b, a); + } + + throw new Error('Cases should have been exhaustive'); +} + +/** + * Check that the directory we were given has all the necessary dependencies in it + * + * It's a warning if this is not true, not an error. + */ +export async function validateAvailableDependencies(directory: string, deps: Record) { + const failures = await Promise.all( + Object.entries(deps).flatMap(async ([name, _dep]) => { + try { + await findDependencyDirectory(name, directory); + return []; + } catch { + return [name]; + } + }), + ); + + if (failures.length > 0) { + logging.warn( + `${directory}: packages necessary to compile examples missing from supplied directory: ${failures.join(', ')}`, + ); + } +} + +/** + * Prepare a temporary directory with symlinks to all the dependencies we need. + * + * - Symlinks the concrete dependencies + * - Tries to first find the symbolic dependencies in a potential monorepo that might be present + * (try both `lerna` and `yarn` monorepos). + * - Installs the remaining symbolic dependencies using 'npm'. + */ +export async function prepareDependencyDirectory(deps: Record): Promise { + const concreteDirs = Object.values(deps) + .filter(isConcrete) + .map((x) => x.resolvedDirectory); + const monorepoPackages = await scanMonoRepos(concreteDirs); + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'rosetta')); + logging.info(`Preparing dependency closure at ${tmpDir}`); + + const packageJson = { + name: 'examples', + version: '0.0.1', + private: true, + dependencies: {} as Record, + }; + + for (const [name, dep] of Object.entries(deps)) { + packageJson.dependencies[name] = + dep.type === 'concrete' + ? `file:${dep.resolvedDirectory}` + : monorepoPackages[name] + ? `file:${monorepoPackages[name]}` + : dep.versionRange; + } + + await fs.writeJson(path.join(tmpDir, 'package.json'), packageJson, { spaces: 2 }); + + // Run 'npm install' on it + cp.execSync('npm install', { cwd: tmpDir, encoding: 'utf-8' }); + + return tmpDir; +} + +/** + * Map package name to directory + */ +async function scanMonoRepos(startingDirs: string[]): Promise> { + const globs = new Set(); + for (const dir of startingDirs) { + // eslint-disable-next-line no-await-in-loop + setExtend(globs, await findMonoRepoGlobs(dir)); + } + + const packageDirectories = await fastGlob(Array.from(globs), { onlyDirectories: true }); + return mkDict( + ( + await Promise.all( + packageDirectories.map(async (directory) => { + const pjLocation = path.join(directory, 'package.json'); + return (await fs.pathExists(pjLocation)) + ? [[(await fs.readJson(pjLocation)).name as string, directory] as const] + : []; + }), + ) + ).flat(), + ); +} + +async function findMonoRepoGlobs(startingDir: string): Promise> { + const ret = new Set(); + + // Lerna monorepo + const lernaJsonDir = await findUp(startingDir, async (dir) => fs.pathExists(path.join(dir, 'lerna.json'))); + if (lernaJsonDir) { + const lernaJson = await fs.readJson(path.join(lernaJsonDir, 'lerna.json')); + for (const glob of lernaJson?.packages ?? []) { + ret.add(path.join(lernaJsonDir, glob)); + } + } + + // Yarn monorepo + const yarnWsDir = await findUp( + startingDir, + async (dir) => + (await fs.pathExists(path.join(dir, 'package.json'))) && + (await fs.readJson(path.join(dir, 'package.json')))?.workspaces !== undefined, + ); + if (yarnWsDir) { + const yarnWs = await fs.readJson(path.join(yarnWsDir, 'package.json')); + for (const glob of yarnWs.workspaces?.packages ?? []) { + ret.add(path.join(yarnWsDir, glob)); + } + } + + return ret; +} + +function isConcrete(x: CompilationDependency): x is Extract { + return x.type === 'concrete'; +} + +function setExtend(xs: Set, ys: Set) { + for (const y of ys) { + xs.add(y); + } + return xs; +} diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts index 3867b36b1e..0504571699 100644 --- a/packages/jsii-rosetta/lib/snippet.ts +++ b/packages/jsii-rosetta/lib/snippet.ts @@ -30,8 +30,22 @@ export interface TypeScriptSnippet { * @default false */ readonly strict?: boolean; + + /** + * Dependencies necessary to compile this snippet + * + * Value is a regular { name -> semver } map like NPM's `dependencies`, + * `devDependencies` etc. + * + * @default none + */ + readonly compilationDependencies?: Record; } +export type CompilationDependency = + | { readonly type: 'concrete'; readonly resolvedDirectory: string } + | { readonly type: 'symbolic'; readonly versionRange: string }; + /** * Description of a location where the snippet is found * @@ -266,6 +280,20 @@ export enum SnippetParameters { */ LITERATE_SOURCE = 'lit', + /** + * This snippet has been infused + * + * This means it has been copied from a different location, and potentially + * even from a different assembly. If so, we can't expect it to compile in + * the future, and if doesn't, we ignore the errors. + * + * N.B: this shouldn't make a difference in normal operation, as the `infuse` + * command will duplicate the translation to the target tablet. This only + * matters if we remove the tablet and try to re-extract an assembly with + * infused examples from somewher else. + */ + INFUSED = 'infused', + /** * What directory to resolve fixtures in for this snippet (system parameter) * diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index 85c515ffa4..40fd85f0d4 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { TargetLanguage } from '../languages'; import * as logging from '../logging'; import { TypeScriptSnippet, SnippetLocation, completeSource } from '../snippet'; -import { mapValues } from '../util'; +import { mapValues, Mutable } from '../util'; import { snippetKey } from './key'; import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './schema'; @@ -282,5 +282,3 @@ export interface Translation { language: string; didCompile?: boolean; } - -type Mutable = { -readonly [P in keyof T]: Mutable }; diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts index ed5a155a3e..abcb973f88 100644 --- a/packages/jsii-rosetta/lib/util.ts +++ b/packages/jsii-rosetta/lib/util.ts @@ -205,3 +205,5 @@ export function isDefined(x: A): x is NonNullable { export function indexBy(xs: A[], fn: (x: A) => string): Record { return mkDict(xs.map((x) => [fn(x), x] as const)); } + +export type Mutable = { -readonly [P in keyof T]: Mutable }; diff --git a/packages/jsii-rosetta/package.json b/packages/jsii-rosetta/package.json index 0749a0a0ed..e7f88a66d2 100644 --- a/packages/jsii-rosetta/package.json +++ b/packages/jsii-rosetta/package.json @@ -22,6 +22,7 @@ "@types/mock-fs": "^4.13.1", "@types/node": "^12.20.37", "@types/workerpool": "^6.1.0", + "@types/semver": "^7.3.9", "eslint": "^8.4.1", "jest": "^27.4.4", "jsii-build-tools": "0.0.0", @@ -39,6 +40,9 @@ "@xmldom/xmldom": "^0.7.5", "workerpool": "^6.1.5", "yargs": "^16.2.0", + "semver": "^7.3.5", + "semver-intersect": "^1.4.0", + "fast-glob": "^3.2.7", "jsii": "0.0.0" }, "license": "Apache-2.0", diff --git a/packages/jsii-rosetta/test/jsii/assemblies.test.ts b/packages/jsii-rosetta/test/jsii/assemblies.test.ts index 97878d9a9f..a5cf9d0d7a 100644 --- a/packages/jsii-rosetta/test/jsii/assemblies.test.ts +++ b/packages/jsii-rosetta/test/jsii/assemblies.test.ts @@ -8,9 +8,9 @@ import { SnippetParameters } from '../../lib/snippet'; import { TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil'; import { fakeAssembly } from './fake-assembly'; -test('Extract snippet from README', () => { +test('Extract snippet from README', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -25,9 +25,9 @@ test('Extract snippet from README', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Extract snippet from submodule READMEs', () => { +test('Extract snippet from submodule READMEs', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ submodules: { @@ -46,9 +46,9 @@ test('Extract snippet from submodule READMEs', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Extract snippet from type docstring', () => { +test('Extract snippet from type docstring', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -72,9 +72,9 @@ test('Extract snippet from type docstring', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Snippet can include fixture', () => { +test('Snippet can include fixture', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -109,9 +109,9 @@ test('Snippet can include fixture', () => { `); }); -test('Use fixture from example', () => { +test('Use fixture from example', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -148,9 +148,9 @@ test('Use fixture from example', () => { expect(snippets[0].visibleSource).toEqual('someExample();'); }); -test('Fixture allows use of import statements', () => { +test('Fixture allows use of import statements', async () => { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ types: { @@ -198,14 +198,14 @@ test('Fixture allows use of import statements', () => { ); }); -test('Backwards compatibility with literate integ tests', () => { +test('Backwards compatibility with literate integ tests', async () => { mockfs({ '/package/test/integ.example.lit.ts': '# Some literate source file', }); try { const snippets = Array.from( - allTypeScriptSnippets([ + await allTypeScriptSnippets([ { assembly: fakeAssembly({ readme: { @@ -262,7 +262,9 @@ test('rosetta fixture from submodule is preferred if it exists', async () => { 'dont pick me\n/// here', ); - const snippets = allTypeScriptSnippets([{ assembly: jsiiModule.assembly, directory: jsiiModule.moduleDirectory }]); + const snippets = await allTypeScriptSnippets([ + { assembly: jsiiModule.assembly, directory: jsiiModule.moduleDirectory }, + ]); expect(snippets[0].completeSource).toMatch(/^pick me/); } finally { diff --git a/yarn.lock b/yarn.lock index 6fd133b9e4..fbc79080ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3592,7 +3592,7 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== -fast-glob@^3.1.1: +fast-glob@^3.1.1, fast-glob@^3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== From 0d576e6229036a6e21b67517d0ef5026dde85b39 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 17 Dec 2021 15:47:38 +0100 Subject: [PATCH 02/16] VEndor file --- packages/jsii-rosetta/vendor/semver-intersect.d.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packages/jsii-rosetta/vendor/semver-intersect.d.ts diff --git a/packages/jsii-rosetta/vendor/semver-intersect.d.ts b/packages/jsii-rosetta/vendor/semver-intersect.d.ts new file mode 100644 index 0000000000..c7f919bcf7 --- /dev/null +++ b/packages/jsii-rosetta/vendor/semver-intersect.d.ts @@ -0,0 +1,13 @@ +/// Hand-written declaration for the semver-intersect module +declare module 'semver-intersect' { + /** + * Computes the intersection between multiple semver ranges. + * + * @param ranges the ranges for which the intersection is requested. + * + * @returns the intersection of `ranges`. + * + * @throws Error if the intersection is empty. + */ + function intersect(...ranges: string[]): string; +} From 360b752e70b7987aff1c91612b19c569e4b2159d Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Mon, 20 Dec 2021 11:10:35 +0100 Subject: [PATCH 03/16] Add tests --- .../test/commands/extract.test.ts | 95 ++++++++++++++++++- packages/jsii-rosetta/test/testutil.ts | 2 +- packages/jsii/lib/helpers.ts | 4 + 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index 74ff8da273..b0fd3ccac8 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -1,4 +1,5 @@ import * as fs from 'fs-extra'; +import { compileJsiiForTest } from 'jsii'; import * as path from 'path'; import { @@ -438,7 +439,7 @@ test('infused examples skip loose mode', async () => { 'index.ts': ` /** * ClassA - * + * * @exampleMetadata lit=integ.test.ts * @example x */ @@ -484,6 +485,98 @@ test('infused examples skip loose mode', async () => { } }); +test('can use additional dependencies from monorepo', async () => { + const asm = await TestJsiiModule.fromSource( + { + 'index.ts': ` + /** + * Class to hold values + * + * @example + * import { ValueHolder } from 'my_assembly'; + * import { SomeClass } from 'otherModule'; + * new ValueHolder(new SomeClass()); + */ + export class ValueHolder { + constructor(public readonly theValue: any) { } + } + `, + }, + { + name: 'my_assembly', + jsii: DUMMY_JSII_CONFIG, + jsiiRosetta: { + exampleDependencies: { + // This relies on the fact that Rosetta will find the package in the monorepo + otherModule: '*', + }, + }, + }, + ); + try { + // GIVEN - install some random other module + await asm.workspace.addDependency( + await compileJsiiForTest( + { + 'index.ts': 'export class SomeClass { }', + }, + { + packageJson: { + name: 'otherModule', + }, + }, + ), + ); + // GIVEN - a lerna.json that would find that package + await fs.writeJson(path.join(asm.workspaceDirectory, 'lerna.json'), { + packages: ['node_modules/*'], + }); + + // WHEN + await extract.extractSnippets([asm.moduleDirectory], defaultExtractOptions); + // THEN -- did not throw an error + } finally { + await asm.cleanup(); + } +}); + +test('can use additional dependencies from NPM', async () => { + const asm = await TestJsiiModule.fromSource( + { + 'index.ts': ` + /** + * Class to hold values + * + * @example + * import { ValueHolder } from 'my_assembly'; + * import { ConstructOrder } from 'constructs'; + * new ValueHolder(ConstructOrder.PREORDER); + */ + export class ValueHolder { + constructor(public readonly theValue: any) { } + } + `, + }, + { + name: 'my_assembly', + jsii: DUMMY_JSII_CONFIG, + jsiiRosetta: { + exampleDependencies: { + // This relies on the fact that Rosetta will find the package in the monorepo + constructs: '^10.0.0', + }, + }, + }, + ); + try { + // WHEN + await extract.extractSnippets([asm.moduleDirectory], defaultExtractOptions); + // THEN -- did not throw an error + } finally { + await asm.cleanup(); + } +}); + class MockTranslator extends RosettaTranslator { public constructor(opts: RosettaTranslatorOptions, translatorFn: jest.Mock) { super(opts); diff --git a/packages/jsii-rosetta/test/testutil.ts b/packages/jsii-rosetta/test/testutil.ts index 953ae3c5fa..1c3011da63 100644 --- a/packages/jsii-rosetta/test/testutil.ts +++ b/packages/jsii-rosetta/test/testutil.ts @@ -35,7 +35,7 @@ export class TestJsiiModule { public readonly moduleDirectory: string; public readonly workspaceDirectory: string; - private constructor(public readonly assembly: spec.Assembly, private readonly workspace: TestWorkspace) { + private constructor(public readonly assembly: spec.Assembly, public readonly workspace: TestWorkspace) { this.moduleDirectory = workspace.dependencyDir(assembly.name); this.workspaceDirectory = workspace.rootDirectory; } diff --git a/packages/jsii/lib/helpers.ts b/packages/jsii/lib/helpers.ts index 37e09040b4..5f5bd579cc 100644 --- a/packages/jsii/lib/helpers.ts +++ b/packages/jsii/lib/helpers.ts @@ -217,11 +217,15 @@ export interface TestCompilationOptions { /** * Parts of projectInfo to override (package name etc) + * + * @deprecated Prefer using `packageJson` instead. */ readonly projectInfo?: Partial; /** * Parts of projectInfo to override (package name etc) + * + * @default - Use some default values */ readonly packageJson?: Partial; } From 49bf75b7420a8f623eb4548ea1dc724227f880d9 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Mon, 20 Dec 2021 13:53:49 +0100 Subject: [PATCH 04/16] Raise test timeouts --- packages/jsii-rosetta/test/commands/extract.test.ts | 2 ++ packages/jsii-rosetta/test/commands/infuse.test.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index b0fd3ccac8..d51744ee1f 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -15,6 +15,8 @@ import { loadAssemblies } from '../../lib/jsii/assemblies'; import { TARGET_LANGUAGES } from '../../lib/languages'; import { TestJsiiModule, DUMMY_JSII_CONFIG, testSnippetLocation } from '../testutil'; +jest.setTimeout(30_000); + const DUMMY_README = ` Here is an example of how to use ClassA: diff --git a/packages/jsii-rosetta/test/commands/infuse.test.ts b/packages/jsii-rosetta/test/commands/infuse.test.ts index 8962d0522f..f8e8b7c316 100644 --- a/packages/jsii-rosetta/test/commands/infuse.test.ts +++ b/packages/jsii-rosetta/test/commands/infuse.test.ts @@ -8,6 +8,8 @@ import { infuse, DEFAULT_INFUSION_RESULTS_NAME } from '../../lib/commands/infuse import { loadAssemblies } from '../../lib/jsii/assemblies'; import { TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil'; +jest.setTimeout(30_000); + const DUMMY_README = ` Here is an example of how to use ClassA: From 710b259abbee789f0093497ef574eec5c4cfa98c Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 22 Dec 2021 13:25:41 +0000 Subject: [PATCH 05/16] Try again with more logging --- packages/@jsii/spec/lib/configuration.ts | 2 ++ packages/jsii-calc/package.json | 2 +- packages/jsii-rosetta/lib/jsii/assemblies.ts | 6 +++++- packages/jsii-rosetta/lib/snippet-dependencies.ts | 12 +++++++++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/@jsii/spec/lib/configuration.ts b/packages/@jsii/spec/lib/configuration.ts index 57f13f5c3e..16355fea82 100644 --- a/packages/@jsii/spec/lib/configuration.ts +++ b/packages/@jsii/spec/lib/configuration.ts @@ -171,6 +171,8 @@ export interface PackageJson { */ devDependencies?: Record; + bundleDependencies?: string[]; + bundledDependencies?: string[]; [key: string]: unknown; diff --git a/packages/jsii-calc/package.json b/packages/jsii-calc/package.json index dc3489e567..0d49f5473d 100644 --- a/packages/jsii-calc/package.json +++ b/packages/jsii-calc/package.json @@ -33,7 +33,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile", + "build": "jsii --project-references --silence-warnings reserved-word && npm run lint && jsii-rosetta --compile --verbose", "watch": "jsii --project-references -w", "lint": "eslint . --ext .js,.ts --ignore-path=.gitignore", "lint:fix": "yarn lint --fix", diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts index 521f149c5c..80cb301151 100644 --- a/packages/jsii-rosetta/lib/jsii/assemblies.ts +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -377,7 +377,11 @@ async function withDependencies(asm: LoadedAssembly, snippet: TypeScriptSnippet) await Promise.all( Object.keys({ ...asm.packageJson?.dependencies, ...asm.packageJson?.peerDependencies }) .filter((name) => !isBuiltinModule(name)) - .filter((name) => !(asm.packageJson?.bundledDependencies ?? []).includes(name)) + .filter( + (name) => + !asm.packageJson?.bundledDependencies?.includes(name) && + !asm.packageJson?.bundleDependencies?.includes(name), + ) .map( async (name) => [ diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index c5b8054deb..7ccf3ae798 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -130,7 +130,13 @@ export async function prepareDependencyDirectory(deps: Record 0) { + logging.debug(`Monorepo package sources: ${Array.from(globs).join(', ')}`); + } + const packageDirectories = await fastGlob(Array.from(globs), { onlyDirectories: true }); return mkDict( ( From e78c46ee837be81c27867c800d6656a83088ed99 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 22 Dec 2021 13:48:24 +0000 Subject: [PATCH 06/16] Do symlink ourselves if we can --- .../jsii-rosetta/lib/snippet-dependencies.ts | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 7ccf3ae798..30e640eb7c 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -111,32 +111,42 @@ export async function prepareDependencyDirectory(deps: Record, - }; - - for (const [name, dep] of Object.entries(deps)) { - packageJson.dependencies[name] = + // Resolved symbolic packages against monorepo + const resolvedDeps = mkDict( + Object.entries(deps).map(([name, dep]) => [ + name, dep.type === 'concrete' - ? `file:${dep.resolvedDirectory}` - : monorepoPackages[name] - ? `file:${monorepoPackages[name]}` - : dep.versionRange; - } + ? dep + : ((monorepoPackages[name] + ? { type: 'concrete', resolvedDirectory: monorepoPackages[name] } + : dep) as CompilationDependency), + ]), + ); - await fs.writeJson(path.join(tmpDir, 'package.json'), packageJson, { spaces: 2 }); + // Use 'npm install' only for the symbolic packages. For the concrete packages, + // npm is going to try and find transitive dependencies as well and it won't know + // about monorepos. + const symbolicInstalls = Object.entries(resolvedDeps).flatMap(([name, dep]) => + isSymbolic(dep) ? [`${name}@${dep.versionRange}`] : [], + ); + const linkedInstalls = mkDict( + Object.entries(resolvedDeps).flatMap(([name, dep]) => + isConcrete(dep) ? [[name, dep.resolvedDirectory] as const] : [], + ), + ); // Run 'npm install' on it - try { - cp.execSync('npm install', { cwd: tmpDir, encoding: 'utf-8' }); - } catch (e) { - logging.error('Failed installing dependency closure from:'); - logging.error(JSON.stringify(packageJson.dependencies, undefined, 2)); - throw e; - } + logging.debug(`Installing example dependencies: ${symbolicInstalls.join(' ')}`); + cp.execSync(`npm install ${symbolicInstalls.join(' ')}`, { cwd: tmpDir, encoding: 'utf-8' }); + + // Symlink the rest + await Promise.all( + Object.entries(linkedInstalls).map(async ([name, source]) => { + const modDir = path.join(tmpDir, 'node_modules'); + await fs.mkdirp(modDir); + await fs.symlink(source, path.join(modDir, name)); + }), + ); return tmpDir; } @@ -199,6 +209,10 @@ async function findMonoRepoGlobs(startingDir: string): Promise> { return ret; } +function isSymbolic(x: CompilationDependency): x is Extract { + return x.type === 'symbolic'; +} + function isConcrete(x: CompilationDependency): x is Extract { return x.type === 'concrete'; } From 684569a5d5581e7c9bcc5d3680715192229b4d8c Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 22 Dec 2021 13:57:07 +0000 Subject: [PATCH 07/16] symlink other way around --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 30e640eb7c..097a08560e 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -144,7 +144,7 @@ export async function prepareDependencyDirectory(deps: Record { const modDir = path.join(tmpDir, 'node_modules'); await fs.mkdirp(modDir); - await fs.symlink(source, path.join(modDir, name)); + await fs.symlink(path.join(modDir, name), source); }), ); From 5255b35a61be9162f4acee112fe7d8c5db81ab7d Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 22 Dec 2021 14:02:45 +0000 Subject: [PATCH 08/16] Nope this is the right way around --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 097a08560e..0504464ee3 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -140,11 +140,11 @@ export async function prepareDependencyDirectory(deps: Record { - const modDir = path.join(tmpDir, 'node_modules'); - await fs.mkdirp(modDir); - await fs.symlink(path.join(modDir, name), source); + await fs.symlink(source, path.join(modDir, name), 'dir'); }), ); From 541846d3b8bf1369b284d9cacd800093d32af5e0 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 23 Dec 2021 10:42:29 +0000 Subject: [PATCH 09/16] Account for packages in scopes --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 0504464ee3..7730d45290 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -141,10 +141,11 @@ export async function prepareDependencyDirectory(deps: Record { - await fs.symlink(source, path.join(modDir, name), 'dir'); + const target = path.join(modDir, name); + await fs.mkdirp(path.dirname(target)); + await fs.symlink(source, target, 'dir'); }), ); From 0bae47570160f4e2201cb7a2c6324d201ed5c4c4 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 23 Dec 2021 13:41:42 +0000 Subject: [PATCH 10/16] Create dir --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 7730d45290..97be89edbc 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -136,6 +136,7 @@ export async function prepareDependencyDirectory(deps: Record Date: Fri, 24 Dec 2021 13:00:34 +0000 Subject: [PATCH 11/16] Skip if unnecessary --- .../jsii-rosetta/lib/snippet-dependencies.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 97be89edbc..0cea176c44 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -137,18 +137,24 @@ export async function prepareDependencyDirectory(deps: Record 0) { + logging.debug(`Installing example dependencies: ${symbolicInstalls.join(' ')}`); + cp.execSync(`npm install ${symbolicInstalls.join(' ')}`, { cwd: tmpDir, encoding: 'utf-8' }); + } // Symlink the rest - const modDir = path.join(tmpDir, 'node_modules'); - await Promise.all( - Object.entries(linkedInstalls).map(async ([name, source]) => { - const target = path.join(modDir, name); - await fs.mkdirp(path.dirname(target)); - await fs.symlink(source, target, 'dir'); - }), - ); + if (Object.keys(linkedInstalls).length > 0) { + logging.debug(`Symlinking example dependencies: ${Object.values(linkedInstalls).join(' ')}`); + const modDir = path.join(tmpDir, 'node_modules'); + await Promise.all( + Object.entries(linkedInstalls).map(async ([name, source]) => { + const target = path.join(modDir, name); + await fs.mkdirp(path.dirname(target)); + await fs.symlink(source, target, 'dir'); + }), + ); + } return tmpDir; } From 65f396ea5b840cc8d17a03cba54835b59fee3590 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 24 Dec 2021 13:42:40 +0000 Subject: [PATCH 12/16] More logging --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 13 +++++++++---- packages/jsii-rosetta/lib/util.ts | 6 ++++++ packages/jsii-rosetta/test/commands/extract.test.ts | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 0cea176c44..680fac7ce5 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -9,7 +9,7 @@ import { intersect } from 'semver-intersect'; import { findDependencyDirectory, findUp } from './find-utils'; import * as logging from './logging'; import { TypeScriptSnippet, CompilationDependency } from './snippet'; -import { mkDict } from './util'; +import { mkDict, formatList } from './util'; /** * Collect the dependencies of a bunch of snippets together in one declaration @@ -169,12 +169,14 @@ async function scanMonoRepos(startingDirs: string[]): Promise 0) { - logging.debug(`Monorepo package sources: ${Array.from(globs).join(', ')}`); + if (globs.size === 0) { + return {}; } + logging.debug(`Monorepo package sources: ${Array.from(globs).join(', ')}`); + const packageDirectories = await fastGlob(Array.from(globs), { onlyDirectories: true }); - return mkDict( + const results = mkDict( ( await Promise.all( packageDirectories.map(async (directory) => { @@ -186,6 +188,9 @@ async function scanMonoRepos(startingDirs: string[]): Promise> { diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts index 93ddc19ad6..6777bcc257 100644 --- a/packages/jsii-rosetta/lib/util.ts +++ b/packages/jsii-rosetta/lib/util.ts @@ -24,6 +24,12 @@ export function printDiagnostics(diags: readonly RosettaDiagnostic[], stream: No } } +export function formatList(xs: string[], n = 5) { + const tooMany = xs.length - n; + + return tooMany > 0 ? `${xs.join(', ')} (and ${tooMany} more)` : xs.join(', '); +} + export const StrictBrand = 'jsii.strict'; interface MaybeStrictDiagnostic { readonly [StrictBrand]?: boolean; diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index c8f58d4386..288bce08a6 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -1,4 +1,5 @@ import * as fs from 'fs-extra'; +import * as logging from '../../lib/logging'; import { compileJsiiForTest } from 'jsii'; import * as path from 'path'; @@ -487,6 +488,7 @@ test('infused examples skip loose mode', async () => { }); test('can use additional dependencies from monorepo', async () => { + logging.configure({ level: logging.Level.VERBOSE }); const asm = await TestJsiiModule.fromSource( { 'index.ts': ` From 455571606533d4434c3c5f4c4c39b5789575b4e8 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 24 Dec 2021 14:29:36 +0000 Subject: [PATCH 13/16] Fix import order --- packages/jsii-rosetta/test/commands/extract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index 288bce08a6..70160f298a 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -1,5 +1,4 @@ import * as fs from 'fs-extra'; -import * as logging from '../../lib/logging'; import { compileJsiiForTest } from 'jsii'; import * as path from 'path'; @@ -14,6 +13,7 @@ import { import * as extract from '../../lib/commands/extract'; import { loadAssemblies } from '../../lib/jsii/assemblies'; import { TARGET_LANGUAGES } from '../../lib/languages'; +import * as logging from '../../lib/logging'; import { TestJsiiModule, DUMMY_JSII_CONFIG, testSnippetLocation } from '../testutil'; jest.setTimeout(30_000); From 3faaa31aca73967ae243316c86ce7761100d8935 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 24 Dec 2021 16:24:34 +0000 Subject: [PATCH 14/16] Thanks fastGlob --- packages/jsii-rosetta/lib/snippet-dependencies.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index 680fac7ce5..b4ea7c38cd 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -175,7 +175,7 @@ async function scanMonoRepos(startingDirs: string[]): Promise(xs: Set, ys: Set) { } return xs; } + +/** + * Necessary for fastGlob + */ +function windowsToUnix(x: string) { + return x.replace(/\\/g, '/'); +} From 2612b83896ab009e8428fdc87cfa94720284faad Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 13 Jan 2022 14:22:55 +0100 Subject: [PATCH 15/16] Review comments --- packages/jsii-rosetta/lib/find-utils.ts | 8 ++------ packages/jsii-rosetta/lib/jsii/assemblies.ts | 2 +- packages/jsii-rosetta/lib/rosetta-translator.ts | 3 +++ packages/jsii-rosetta/lib/snippet-dependencies.ts | 11 ++++++----- packages/jsii-rosetta/lib/util.ts | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/jsii-rosetta/lib/find-utils.ts b/packages/jsii-rosetta/lib/find-utils.ts index 807aa334f5..77c29907cd 100644 --- a/packages/jsii-rosetta/lib/find-utils.ts +++ b/packages/jsii-rosetta/lib/find-utils.ts @@ -51,10 +51,10 @@ export function findUp(directory: string, pred: (dir: string) => boolean): strin // eslint-disable-next-line @typescript-eslint/promise-function-async export function findUp( directory: string, - pred: ((dir: string) => boolean) | ((dir: string) => Promise), + pred: (dir: string) => boolean | Promise, ): Promise | string | undefined { const result = pred(directory); - if (isPromise(result)) { + if (result instanceof Promise) { return result.then((thisDirectory) => (thisDirectory ? directory : recurse())); } @@ -69,10 +69,6 @@ export function findUp( } } -function isPromise(x: A | Promise): x is Promise { - return typeof x === 'object' && (x as any).then; -} - /** * Whether the given dependency is a built-in * diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts index 80cb301151..f9edbb06a6 100644 --- a/packages/jsii-rosetta/lib/jsii/assemblies.ts +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -216,7 +216,7 @@ export async function allTypeScriptSnippets( } }) .map(async ({ snippet, loaded }) => { - const isInfused = snippet.parameters?.infused !== undefined; + const isInfused = snippet.parameters?.infused != null; // Ignore fixturization errors if requested on this command, or if the snippet was infused const ignoreFixtureErrors = loose || isInfused; diff --git a/packages/jsii-rosetta/lib/rosetta-translator.ts b/packages/jsii-rosetta/lib/rosetta-translator.ts index 08ec54beee..e6158ba3fa 100644 --- a/packages/jsii-rosetta/lib/rosetta-translator.ts +++ b/packages/jsii-rosetta/lib/rosetta-translator.ts @@ -203,6 +203,9 @@ export interface ReadFromCacheResults { } export interface TranslateAllOptions { + /** + * @default - Create a temporary directory with all necessary packages + */ readonly compilationDirectory?: string; /** diff --git a/packages/jsii-rosetta/lib/snippet-dependencies.ts b/packages/jsii-rosetta/lib/snippet-dependencies.ts index b4ea7c38cd..a21ace6710 100644 --- a/packages/jsii-rosetta/lib/snippet-dependencies.ts +++ b/packages/jsii-rosetta/lib/snippet-dependencies.ts @@ -136,8 +136,6 @@ export async function prepareDependencyDirectory(deps: Record 0) { logging.debug(`Installing example dependencies: ${symbolicInstalls.join(' ')}`); cp.execSync(`npm install ${symbolicInstalls.join(' ')}`, { cwd: tmpDir, encoding: 'utf-8' }); @@ -150,8 +148,11 @@ export async function prepareDependencyDirectory(deps: Record { const target = path.join(modDir, name); - await fs.mkdirp(path.dirname(target)); - await fs.symlink(source, target, 'dir'); + if (!(await fs.pathExists(target))) { + // Package could be namespaced, so ensure the namespace dir exists + await fs.mkdirp(path.dirname(target)); + await fs.symlink(source, target, 'dir'); + } }), ); } @@ -162,7 +163,7 @@ export async function prepareDependencyDirectory(deps: Record> { +async function scanMonoRepos(startingDirs: readonly string[]): Promise> { const globs = new Set(); for (const dir of startingDirs) { // eslint-disable-next-line no-await-in-loop diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts index 6777bcc257..fe6ca7832b 100644 --- a/packages/jsii-rosetta/lib/util.ts +++ b/packages/jsii-rosetta/lib/util.ts @@ -27,7 +27,7 @@ export function printDiagnostics(diags: readonly RosettaDiagnostic[], stream: No export function formatList(xs: string[], n = 5) { const tooMany = xs.length - n; - return tooMany > 0 ? `${xs.join(', ')} (and ${tooMany} more)` : xs.join(', '); + return tooMany > 0 ? `${xs.slice(0, n).join(', ')} (and ${tooMany} more)` : xs.join(', '); } export const StrictBrand = 'jsii.strict'; From 6c3d25581485e0dc0c3460ad7caca57ef14dafd1 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 13 Jan 2022 14:45:39 +0100 Subject: [PATCH 16/16] sync -> async --- packages/jsii-rosetta/lib/commands/coverage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-rosetta/lib/commands/coverage.ts b/packages/jsii-rosetta/lib/commands/coverage.ts index d7fb57b0d9..d223e41262 100644 --- a/packages/jsii-rosetta/lib/commands/coverage.ts +++ b/packages/jsii-rosetta/lib/commands/coverage.ts @@ -7,7 +7,7 @@ export async function checkCoverage(assemblyLocations: readonly string[]): Promi logging.info(`Loading ${assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(assemblyLocations, false); - const snippets = Array.from(allTypeScriptSnippets(assemblies, true)); + const snippets = Array.from(await allTypeScriptSnippets(assemblies, true)); const translator = new RosettaTranslator({ assemblies: assemblies.map((a) => a.assembly),