feat: cache npm metadata

.yarn/versions/07d34d58.yml

@@ -0,0 +1,39 @@
+releases:
+  "@yarnpkg/cli": minor
+  "@yarnpkg/core": minor
+  "@yarnpkg/fslib": minor
+  "@yarnpkg/plugin-npm": minor
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/libzip"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnp"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"
+  - "@yarnpkg/shell"

CHANGELOG.md

@@ -84,6 +84,7 @@ The following changes only affect people writing Yarn plugins:
 
 ### Installs
 
+- Yarn now caches npm version metadata, leading to faster resolution steps and decreased network data usage.
 - The `pnpm` linker avoids creating symlinks that lead to loops on the file system, by moving them higher up in the directory structure.
 - The `pnpm` linker no longer reports duplicate "incompatible virtual" warnings.
 

packages/acceptance-tests/pkg-tests-specs/sources/commands/stage.test.js

@@ -48,6 +48,7 @@ describe(`Commands`, () => {
         await expect(run(`stage`, `-n`, {cwd: path})).resolves.toMatchObject({
           stdout: [
             `${npath.fromPortablePath(`${path}/.pnp.cjs`)}\n`,
+            `${npath.fromPortablePath(`${path}/.yarn/global/metadata/npm/b98544/localhost/no-deps.json`)}\n`,
             `${npath.fromPortablePath(`${path}/.yarn/global/cache/no-deps-npm-1.0.0-cf533b267a-0.zip`)}\n`,
             `${npath.fromPortablePath(`${path}/.yarn/cache/.gitignore`)}\n`,
             `${npath.fromPortablePath(`${path}/.yarn/cache/no-deps-npm-1.0.0-cf533b267a-e0e60294c2.zip`)}\n`,

packages/plugin-npm/package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@yarnpkg/fslib": "workspace:^",
     "enquirer": "^2.3.6",
+    "lodash": "^4.17.15",
     "semver": "^7.1.2",
     "ssri": "^6.0.1",
     "tslib": "^2.4.0"
@@ -20,6 +21,7 @@
     "@yarnpkg/plugin-pack": "workspace:^"
   },
   "devDependencies": {
+    "@types/lodash": "^4.14.136",
     "@types/semver": "^7.1.0",
     "@types/ssri": "^6.0.1",
     "@yarnpkg/core": "workspace:^",

packages/plugin-npm/sources/NpmSemverResolver.ts

@@ -47,11 +47,9 @@ export class NpmSemverResolver implements Resolver {
     if (range === null)
       throw new Error(`Expected a valid range, got ${descriptor.range.slice(PROTOCOL.length)}`);
 
-    const registryData = await npmHttpUtils.get(npmHttpUtils.getIdentUrl(descriptor), {
-      customErrorMessage: npmHttpUtils.customPackageError,
-      configuration: opts.project.configuration,
-      ident: descriptor,
-      jsonResponse: true,
+    const registryData = await npmHttpUtils.getPackageMetadata(descriptor, {
+      project: opts.project,
+      version: semver.valid(range.raw) ? range.raw : undefined,
     });
 
     const candidates = miscUtils.mapAndFilter(Object.keys(registryData.versions), version => {
@@ -127,11 +125,9 @@ export class NpmSemverResolver implements Resolver {
     if (version === null)
       throw new ReportError(MessageName.RESOLVER_NOT_FOUND, `The npm semver resolver got selected, but the version isn't semver`);
 
-    const registryData = await npmHttpUtils.get(npmHttpUtils.getIdentUrl(locator), {
-      customErrorMessage: npmHttpUtils.customPackageError,
-      configuration: opts.project.configuration,
-      ident: locator,
-      jsonResponse: true,
+    const registryData = await npmHttpUtils.getPackageMetadata(locator, {
+      project: opts.project,
+      version,
     });
 
     if (!Object.prototype.hasOwnProperty.call(registryData, `versions`))

packages/plugin-npm/sources/NpmTagResolver.ts

@@ -39,10 +39,8 @@ export class NpmTagResolver implements Resolver {
   async getCandidates(descriptor: Descriptor, dependencies: unknown, opts: ResolveOptions) {
     const tag = descriptor.range.slice(PROTOCOL.length);
 
-    const registryData = await npmHttpUtils.get(npmHttpUtils.getIdentUrl(descriptor), {
-      configuration: opts.project.configuration,
-      ident: descriptor,
-      jsonResponse: true,
+    const registryData = await npmHttpUtils.getPackageMetadata(descriptor, {
+      project: opts.project,
     });
 
     if (!Object.prototype.hasOwnProperty.call(registryData, `dist-tags`))

packages/plugin-npm/sources/npmHttpUtils.ts

@@ -1,11 +1,13 @@
-import {Configuration, Ident, formatUtils, httpUtils, nodeUtils, StreamReport} from '@yarnpkg/core';
-import {MessageName, ReportError}                                              from '@yarnpkg/core';
-import {prompt}                                                                from 'enquirer';
-import {URL}                                                                   from 'url';
+import {Configuration, Ident, formatUtils, httpUtils, nodeUtils, StreamReport, structUtils, IdentHash, hashUtils, Project, miscUtils} from '@yarnpkg/core';
+import {MessageName, ReportError}                                                                                                     from '@yarnpkg/core';
+import {Filename, PortablePath, ppath, toFilename, xfs}                                                                               from '@yarnpkg/fslib';
+import {prompt}                                                                                                                       from 'enquirer';
+import pick                                                                                                                           from 'lodash/pick';
+import {URL}                                                                                                                          from 'url';
 
-import {Hooks}                                                                 from './index';
-import * as npmConfigUtils                                                     from './npmConfigUtils';
-import {MapLike}                                                               from './npmConfigUtils';
+import {Hooks}                                                                                                                        from './index';
+import * as npmConfigUtils                                                                                                            from './npmConfigUtils';
+import {MapLike}                                                                                                                      from './npmConfigUtils';
 
 export enum AuthType {
   NO_AUTH,
@@ -33,7 +35,7 @@ export type Options = httpUtils.Options & RegistryOptions & {
  * It doesn't handle 403 Forbidden, as the npm registry uses it when the user attempts
  * a prohibited action, such as publishing a package with a similar name to an existing package.
  */
-export async function handleInvalidAuthenticationError(error: any, {attemptedAs, registry, headers, configuration}: {attemptedAs?: string, registry: string, headers: {[key: string]: string} | undefined, configuration: Configuration}) {
+export async function handleInvalidAuthenticationError(error: any, {attemptedAs, registry, headers, configuration}: {attemptedAs?: string, registry: string, headers: {[key: string]: string | undefined} | undefined, configuration: Configuration}) {
   if (isOtpError(error))
     throw new ReportError(MessageName.AUTHENTICATION_INVALID, `Invalid OTP token`);
 
@@ -64,15 +66,169 @@ export function getIdentUrl(ident: Ident) {
   }
 }
 
+export type GetPackageMetadataOptions = Omit<Options, 'ident' | 'configuration'> & {
+  project: Project;
+
+  /**
+   * Warning: This option will return all cached metadata if the version is found, but the rest of the metadata can be stale.
+   */
+  version?: string;
+};
+
+// We use 2 different caches:
+// - an in-memory cache, to avoid hitting the disk and the network more than once per process for each package
+// - an on-disk cache, for exact version matches and to avoid refetching the metadata if the resource hasn't changed on the server
+
+const PACKAGE_METADATA_CACHE = new Map<IdentHash, Promise<PackageMetadata> | PackageMetadata>();
+
+/**
+ * Caches and returns the package metadata for the given ident.
+ *
+ * Note: This function only caches and returns specific fields from the metadata.
+ * If you need other fields, use the uncached {@link get} or consider whether it would make more sense to extract
+ * the fields from the on-disk packages using the linkers or from the fetch results using the fetchers.
+ */
+export async function getPackageMetadata(ident: Ident, {project, registry, headers, version, ...rest}: GetPackageMetadataOptions): Promise<PackageMetadata> {
+  return await miscUtils.getFactoryWithDefault(PACKAGE_METADATA_CACHE, ident.identHash, async () => {
+    const {configuration} = project;
+
+    registry = normalizeRegistry(configuration, {ident, registry});
+
+    const registryFolder = getRegistryFolder(configuration, registry);
+    const identPath = ppath.join(registryFolder, `${structUtils.slugifyIdent(ident)}.json`);
+
+    let cached: CachedMetadata | null = null;
+
+    // We bypass the on-disk cache for security reasons if the lockfile needs to be refreshed,
+    // since most likely the user is trying to validate the metadata using hardened mode.
+    if (!project.lockfileNeedsRefresh) {
+      try {
+        cached = await xfs.readJsonPromise(identPath) as CachedMetadata;
+
+        if (typeof version !== `undefined` && typeof cached.metadata.versions[version] !== `undefined`) {
+          return cached.metadata;
+        }
+      } catch {}
+    }
+
+    return await get(getIdentUrl(ident), {
+      ...rest,
+      customErrorMessage: customPackageError,
+      configuration,
+      registry,
+      ident,
+      headers: {
+        ...headers,
+        // We set both headers in case a registry doesn't support ETags
+        [`If-None-Match`]: cached?.etag,
+        [`If-Modified-Since`]: cached?.lastModified,
+      },
+      wrapNetworkRequest: async executor => async () => {
+        const response = await executor();
+
+        if (response.statusCode === 304) {
+          if (cached === null)
+            throw new Error(`Assertion failed: cachedMetadata should not be null`);
+
+          return {
+            ...response,
+            body: cached.metadata,
+          };
+        }
+
+        const packageMetadata = pickPackageMetadata(JSON.parse(response.body.toString()));
+
+        PACKAGE_METADATA_CACHE.set(ident.identHash, packageMetadata);
+
+        const metadata: CachedMetadata = {
+          metadata: packageMetadata,
+          etag: response.headers.etag,
+          lastModified: response.headers[`last-modified`],
+        };
+
+        // We append the PID because it is guaranteed that this code is only run once per process for a given ident
+        const identPathTemp = `${identPath}-${process.pid}.tmp` as PortablePath;
+
+        await xfs.mkdirPromise(registryFolder, {recursive: true});
+        await xfs.writeJsonPromise(identPathTemp, metadata, {compact: true});
+
+        // Doing a rename is important to ensure the cache is atomic
+        await xfs.renamePromise(identPathTemp, identPath);
+
+        return {
+          ...response,
+          body: packageMetadata,
+        };
+      },
+    });
+  });
+}
+
+type CachedMetadata = {
+  metadata: PackageMetadata;
+  etag?: string;
+  lastModified?: string;
+};
+
+export type PackageMetadata = {
+  'dist-tags': Record<string, string>;
+  versions: Record<string, any>;
+};
+
+const CACHED_FIELDS = [
+  `name`,
+
+  `dist.tarball`,
+
+  `bin`,
+  `scripts`,
+
+  `os`,
+  `cpu`,
+  `libc`,
+
+  `dependencies`,
+  `dependenciesMeta`,
+  `optionalDependencies`,
+
+  `peerDependencies`,
+  `peerDependenciesMeta`,
+];
+
+function pickPackageMetadata(metadata: PackageMetadata): PackageMetadata {
+  return {
+    'dist-tags': metadata[`dist-tags`],
+    versions: Object.fromEntries(Object.entries(metadata.versions).map(([key, value]) => [
+      key,
+      pick(value, CACHED_FIELDS),
+    ])),
+  };
+}
+
+/**
+ * Used to invalidate the on-disk cache when the format changes.
+ */
+const CACHE_KEY = hashUtils.makeHash(...CACHED_FIELDS).slice(0, 6);
+
+function getRegistryFolder(configuration: Configuration, registry: string) {
+  const metadataFolder = getMetadataFolder(configuration);
+
+  const parsed = new URL(registry);
+  const registryFilename = toFilename(parsed.hostname);
+
+  return ppath.join(metadataFolder, CACHE_KEY as Filename, registryFilename);
+}
+
+function getMetadataFolder(configuration: Configuration) {
+  return ppath.join(configuration.get(`globalFolder`), `metadata/npm`);
+}
+
 export async function get(path: string, {configuration, headers, ident, authType, registry, ...rest}: Options) {
-  if (ident && typeof registry === `undefined`)
-    registry = npmConfigUtils.getScopeRegistry(ident.scope, {configuration});
+  registry = normalizeRegistry(configuration, {ident, registry});
+
   if (ident && ident.scope && typeof authType === `undefined`)
     authType = AuthType.BEST_EFFORT;
 
-  if (typeof registry !== `string`)
-    throw new Error(`Assertion failed: The registry should be a string`);
-
   const auth = await getAuthenticationHeader(registry, {authType, configuration, ident});
   if (auth)
     headers = {...headers, authorization: auth};
@@ -87,11 +243,7 @@ export async function get(path: string, {configuration, headers, ident, authType
 }
 
 export async function post(path: string, body: httpUtils.Body, {attemptedAs, configuration, headers, ident, authType = AuthType.ALWAYS_AUTH, registry, otp, ...rest}: Options & {attemptedAs?: string}) {
-  if (ident && typeof registry === `undefined`)
-    registry = npmConfigUtils.getScopeRegistry(ident.scope, {configuration});
-
-  if (typeof registry !== `string`)
-    throw new Error(`Assertion failed: The registry should be a string`);
+  registry = normalizeRegistry(configuration, {ident, registry});
 
   const auth = await getAuthenticationHeader(registry, {authType, configuration, ident});
   if (auth)
@@ -123,11 +275,7 @@ export async function post(path: string, body: httpUtils.Body, {attemptedAs, con
 }
 
 export async function put(path: string, body: httpUtils.Body, {attemptedAs, configuration, headers, ident, authType = AuthType.ALWAYS_AUTH, registry, otp, ...rest}: Options & {attemptedAs?: string}) {
-  if (ident && typeof registry === `undefined`)
-    registry = npmConfigUtils.getScopeRegistry(ident.scope, {configuration});
-
-  if (typeof registry !== `string`)
-    throw new Error(`Assertion failed: The registry should be a string`);
+  registry = normalizeRegistry(configuration, {ident, registry});
 
   const auth = await getAuthenticationHeader(registry, {authType, configuration, ident});
   if (auth)
@@ -159,11 +307,7 @@ export async function put(path: string, body: httpUtils.Body, {attemptedAs, conf
 }
 
 export async function del(path: string, {attemptedAs, configuration, headers, ident, authType = AuthType.ALWAYS_AUTH, registry, otp, ...rest}: Options & {attemptedAs?: string}) {
-  if (ident && typeof registry === `undefined`)
-    registry = npmConfigUtils.getScopeRegistry(ident.scope, {configuration});
-
-  if (typeof registry !== `string`)
-    throw new Error(`Assertion failed: The registry should be a string`);
+  registry = normalizeRegistry(configuration, {ident, registry});
 
   const auth = await getAuthenticationHeader(registry, {authType, configuration, ident});
   if (auth)
@@ -194,6 +338,16 @@ export async function del(path: string, {attemptedAs, configuration, headers, id
   }
 }
 
+function normalizeRegistry(configuration: Configuration, {ident, registry}: Partial<RegistryOptions>): string {
+  if (typeof registry === `undefined` && ident)
+    return npmConfigUtils.getScopeRegistry(ident.scope, {configuration});
+
+  if (typeof registry !== `string`)
+    throw new Error(`Assertion failed: The registry should be a string`);
+
+  return registry;
+}
+
 async function getAuthenticationHeader(registry: string, {authType = AuthType.CONFIGURATION, configuration, ident}: {authType?: AuthType, configuration: Configuration, ident: RegistryOptions['ident']}) {
   const effectiveConfiguration = npmConfigUtils.getAuthConfiguration(registry, {configuration, ident});
   const mustAuthenticate = shouldAuthenticate(effectiveConfiguration, authType);
@@ -242,7 +396,7 @@ function shouldAuthenticate(authConfiguration: MapLike, authType: AuthType) {
   }
 }
 
-async function whoami(registry: string, headers: {[key: string]: string} | undefined, {configuration}: {configuration: Configuration}) {
+async function whoami(registry: string, headers: {[key: string]: string | undefined} | undefined, {configuration}: {configuration: Configuration}) {
   if (typeof headers === `undefined` || typeof headers.authorization === `undefined`)
     return `an anonymous user`;
 

packages/yarnpkg-core/sources/Plugin.ts

@@ -91,9 +91,9 @@ export interface Hooks {
    * add some logging.
    */
   wrapNetworkRequest?: (
-    executor: () => Promise<any>,
+    executor: () => Promise<httpUtils.Response>,
     extra: WrapNetworkRequestInfo
-  ) => Promise<() => Promise<any>>;
+  ) => Promise<() => Promise<httpUtils.Response>>;
 
   /**
    * Called before the build, to compute a global hash key that we will use