From 2bc9e84edd05a27e4173948a25752020371495b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Jona=C5=A1?= Date: Thu, 17 Nov 2022 14:04:50 +0100 Subject: [PATCH] feat(core): support npm v1 lock file pruning with disclaimer (#13218) --- e2e/vite/src/vite.test.ts | 10 +- .../utils/lock-file/__fixtures__/npm.lock.ts | 108 +++++++++--------- packages/nx/src/utils/lock-file/npm.spec.ts | 77 +++++++++---- packages/nx/src/utils/lock-file/npm.ts | 59 ++++++++-- 4 files changed, 160 insertions(+), 94 deletions(-) diff --git a/e2e/vite/src/vite.test.ts b/e2e/vite/src/vite.test.ts index bda29e8502dca..a47ab063dea03 100644 --- a/e2e/vite/src/vite.test.ts +++ b/e2e/vite/src/vite.test.ts @@ -135,9 +135,13 @@ describe('Vite Plugin', () => { afterEach(() => killPorts()); it('should serve applications in dev mode', async () => { - const p = await runCommandUntil(`run ${myApp}:serve`, (output) => { - return output.includes('Local:'); - }); + const port = 4212; + const p = await runCommandUntil( + `run ${myApp}:serve --port=${port}`, + (output) => { + return output.includes('Local:'); + } + ); p.kill(); }, 200000); }); diff --git a/packages/nx/src/utils/lock-file/__fixtures__/npm.lock.ts b/packages/nx/src/utils/lock-file/__fixtures__/npm.lock.ts index ec2772c90d21b..4e58f980f1850 100644 --- a/packages/nx/src/utils/lock-file/__fixtures__/npm.lock.ts +++ b/packages/nx/src/utils/lock-file/__fixtures__/npm.lock.ts @@ -21565,34 +21565,13 @@ export const lockFileV1YargsAndDevkitOnly = `{ "peer": true }, "@yarnpkg/parsers": { - "version": "3.0.0-rc.28", - "resolved": "https://registry.npmjs.org/@yarnpkg/parsers/-/parsers-3.0.0-rc.28.tgz", - "integrity": "sha512-OdBYBaACPjFnqek4jtyR5VH7wX5i7BwfS0AP8m6hTqgULRVOLEc6TKxUBxMCTISzZPGdo5wWAB7OcMmU6G2UnA==", + "version": "3.0.0-rc.27", + "resolved": "https://registry.npmjs.org/@yarnpkg/parsers/-/parsers-3.0.0-rc.27.tgz", + "integrity": "sha512-qs2wZulOYVjaOS6tYOs3SsR7m/qeHwjPrB5i4JtBJELsgWrEkyL+rJH21RA+fVwttJobAYQqw5Xj5SYLaDK/bQ==", "peer": true, "requires": { "js-yaml": "^3.10.0", "tslib": "^2.4.0" - }, - "dependencies": { - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "peer": true, - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "peer": true, - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - } } }, "@zkochan/js-yaml": { @@ -21602,6 +21581,14 @@ export const lockFileV1YargsAndDevkitOnly = `{ "peer": true, "requires": { "argparse": "^2.0.1" + }, + "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "peer": true + } } }, "ansi-colors": { @@ -21634,10 +21621,13 @@ export const lockFileV1YargsAndDevkitOnly = `{ } }, "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "peer": true + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "peer": true, + "requires": { + "sprintf-js": "~1.0.2" + } }, "async": { "version": "3.2.4", @@ -21718,9 +21708,9 @@ export const lockFileV1YargsAndDevkitOnly = `{ } }, "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -22127,21 +22117,13 @@ export const lockFileV1YargsAndDevkitOnly = `{ } }, "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "peer": true, - "requires": { - "argparse": "^2.0.1" - } - }, - "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "peer": true, "requires": { - "minimist": "^1.2.0" + "argparse": "^1.0.7", + "esprima": "^4.0.0" } }, "jsonc-parser": { @@ -22289,14 +22271,19 @@ export const lockFileV1YargsAndDevkitOnly = `{ "yargs-parser": "21.1.1" }, "dependencies": { - "chalk": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "peer": true + }, + "js-yaml": { "version": "4.1.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", - "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "peer": true, "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" + "argparse": "^2.0.1" } }, "minimatch": { @@ -22481,12 +22468,6 @@ export const lockFileV1YargsAndDevkitOnly = `{ "ansi-regex": "^5.0.1" } }, - "strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", - "peer": true - }, "strong-log-transformer": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/strong-log-transformer/-/strong-log-transformer-2.1.0.tgz", @@ -22553,6 +22534,23 @@ export const lockFileV1YargsAndDevkitOnly = `{ "json5": "^1.0.1", "minimist": "^1.2.6", "strip-bom": "^3.0.0" + }, + "dependencies": { + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "peer": true, + "requires": { + "minimist": "^1.2.0" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "peer": true + } } }, "tslib": { diff --git a/packages/nx/src/utils/lock-file/npm.spec.ts b/packages/nx/src/utils/lock-file/npm.spec.ts index 2b5d2975e4cfa..8465ff099c467 100644 --- a/packages/nx/src/utils/lock-file/npm.spec.ts +++ b/packages/nx/src/utils/lock-file/npm.spec.ts @@ -14,6 +14,19 @@ import { lockFileV1YargsAndDevkitOnly, lockFileV2YargsAndDevkitOnly, } from './__fixtures__/npm.lock'; +import { vol } from 'memfs'; +import { readJsonFile } from '../fileutils'; + +jest.mock('fs', () => require('memfs').fs); + +jest.mock('@nrwl/devkit', () => ({ + ...jest.requireActual('@nrwl/devkit'), + workspaceRoot: '/root', +})); + +jest.mock('nx/src/utils/workspace-root', () => ({ + workspaceRoot: '/root', +})); describe('npm LockFile utility', () => { describe('v3', () => { @@ -316,32 +329,48 @@ describe('npm LockFile utility', () => { expect(stringifyNpmLockFile(parsedLockFile)).toEqual(lockFileV1); }); - xit('should prune the lock file', () => { - expect( - Object.keys( - pruneNpmLockFile(parsedLockFile, ['typescript']).dependencies - ).length - ).toEqual(1); - expect( - Object.keys( - pruneNpmLockFile(parsedLockFile, ['yargs', '@nrwl/devkit']) - .dependencies - ).length - ).toEqual(136); - }); + describe('pruning', () => { + beforeAll(() => { + const v2packages = JSON.parse(lockFileV2).packages; + const fileSys = {}; + // map all v2 packages to the file system + Object.keys(v2packages).forEach((key) => { + if (key) { + fileSys[`/root/${key}/package.json`] = JSON.stringify( + v2packages[key] + ); + } + }); + vol.fromJSON(fileSys, '/root'); + }); - xit('should correctly prune lockfile with single package', () => { - expect( - stringifyNpmLockFile(pruneNpmLockFile(parsedLockFile, ['typescript'])) - ).toEqual(lockFileV1JustTypescript); - }); + it('should prune the lock file', () => { + expect( + Object.keys( + pruneNpmLockFile(parsedLockFile, ['typescript']).dependencies + ).length + ).toEqual(1); + expect( + Object.keys( + pruneNpmLockFile(parsedLockFile, ['yargs', '@nrwl/devkit']) + .dependencies + ).length + ).toEqual(136); + }); - xit('should correctly prune lockfile with multiple packages', () => { - expect( - stringifyNpmLockFile( - pruneNpmLockFile(parsedLockFile, ['yargs', '@nrwl/devkit']) - ) - ).toEqual(lockFileV1YargsAndDevkitOnly); + it('should correctly prune lockfile with single package', () => { + expect( + stringifyNpmLockFile(pruneNpmLockFile(parsedLockFile, ['typescript'])) + ).toEqual(lockFileV1JustTypescript); + }); + + it('should correctly prune lockfile with multiple packages', () => { + expect( + stringifyNpmLockFile( + pruneNpmLockFile(parsedLockFile, ['yargs', '@nrwl/devkit']) + ) + ).toEqual(lockFileV1YargsAndDevkitOnly); + }); }); }); }); diff --git a/packages/nx/src/utils/lock-file/npm.ts b/packages/nx/src/utils/lock-file/npm.ts index 1deedd87678fb..0bd366af7e9b9 100644 --- a/packages/nx/src/utils/lock-file/npm.ts +++ b/packages/nx/src/utils/lock-file/npm.ts @@ -1,4 +1,9 @@ +import { existsSync } from 'fs'; import { satisfies } from 'semver'; +import { readJsonFile } from '../fileutils'; +import { output } from '../output'; +import { joinPathFragments } from '../path'; +import { workspaceRoot } from '../workspace-root'; import { LockFileData, PackageDependency } from './lock-file-type'; import { sortObject, @@ -450,16 +455,26 @@ export function pruneNpmLockFile( packages: string[], projectName?: string ): LockFileData { + let isV1; + // NPM V1 does not track full dependency list in the lock file, // so we can't reuse the lock file to generate a new one if (lockFileData.lockFileMetadata.metadata.lockfileVersion === 1) { - console.warn( - `npm v7 is required to prune lockfile. Please upgrade to npm v7 or run "npm i --package-lock-only" to generate pruned lockfile. -Returning entire lock file.` - ); - return lockFileData; + output.warn({ + title: 'Pruning v1 lock file', + bodyLines: [ + `If your "node_modules" are not in sync with the lock file, you might get inaccurate results.`, + `Run "npm ci" to ensure your installed packages are synchronized or upgrade to NPM v7+ to benefit from the new lock file format`, + ], + }); + isV1 = true; } - const dependencies = pruneDependencies(lockFileData.dependencies, packages); + + const dependencies = pruneDependencies( + lockFileData.dependencies, + packages, + isV1 + ); const lockFileMetadata = { ...lockFileData.lockFileMetadata, ...pruneRootPackage(lockFileData, packages, projectName), @@ -503,7 +518,8 @@ function pruneRootPackage( // iterate over packages to collect the affected tree of dependencies function pruneDependencies( dependencies: LockFileData['dependencies'], - packages: string[] + packages: string[], + isV1?: boolean ): LockFileData['dependencies'] { const result: LockFileData['dependencies'] = {}; @@ -523,7 +539,8 @@ function pruneDependencies( [packageName], dependencies, result, - result[packageName][key] + result[packageName][key], + isV1 ); } else { console.warn( @@ -543,17 +560,34 @@ function pruneTransitiveDependencies( dependencies: LockFileData['dependencies'], prunedDeps: LockFileData['dependencies'], value: PackageDependency, + isV1?: boolean, modifier?: 'dev' | 'optional' | 'peer' ): void { - if (!value.dependencies && !value.peerDependencies) { + let packageJSON: PackageDependency; + if (isV1) { + const pathToPackageJSON = joinPathFragments( + workspaceRoot, + value.packageMeta[0].path, + 'package.json' + ); + // if node_modules are our of sync with lock file, we might not have the package.json + if (existsSync(pathToPackageJSON)) { + packageJSON = readJsonFile(pathToPackageJSON); + } + } + if ( + !value.dependencies && + !value.peerDependencies && + !packageJSON?.peerDependencies + ) { return; } Object.entries({ ...value.dependencies, - ...value.devDependencies, ...value.peerDependencies, ...value.optionalDependencies, + ...packageJSON?.peerDependencies, }).forEach(([packageName, version]: [string, string]) => { const versions = dependencies[packageName]; if (versions) { @@ -579,7 +613,7 @@ function pruneTransitiveDependencies( const packageMeta = setPackageMetaModifiers( packageName, dependency, - value, + packageJSON || value, modifier ); currentMeta.push(packageMeta); @@ -589,7 +623,7 @@ function pruneTransitiveDependencies( const packageMeta = setPackageMetaModifiers( packageName, dependency, - value, + packageJSON || value, modifier ); @@ -601,6 +635,7 @@ function pruneTransitiveDependencies( dependencies, prunedDeps, prunedDeps[packageName][key], + isV1, getModifier(packageMeta) ); }