From 693106109c98e617e53345365da2ace3c8e9f4d2 Mon Sep 17 00:00:00 2001 From: James Gill Date: Sat, 15 Oct 2022 12:07:01 +0100 Subject: [PATCH] feat: yarn lock v2 support plus improvements This adds support for yarn.lock v2, that yaml-like format. This means initial support for yarn2 and yarn3 as at time of writing they use this version. Additionally support for depgraph for yarn lock v1 (Yarn1) has been added which should improve performance for these older projects. --- package-lock.json | 14 ++-- package.json | 2 +- src/lib/plugins/nodejs-plugin/index.ts | 23 +++++- .../plugins/nodejs-plugin/npm-lock-parser.ts | 76 ++++++++++++++++++- .../acceptance/snyk-test/all-projects.spec.ts | 4 +- .../cli-monitor.all-projects.spec.ts | 2 +- test/tap/cli-test/cli-test.yarn.spec.ts | 2 +- 7 files changed, 104 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8a291511124..f7a7c7e6749 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,7 +71,7 @@ "snyk-gradle-plugin": "3.24.6", "snyk-module": "3.1.0", "snyk-mvn-plugin": "2.32.0", - "snyk-nodejs-lockfile-parser": "1.44.0", + "snyk-nodejs-lockfile-parser": "1.45.1", "snyk-nuget-plugin": "1.23.5", "snyk-php-plugin": "1.9.2", "snyk-policy": "^1.25.0", @@ -17128,9 +17128,9 @@ "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "node_modules/snyk-nodejs-lockfile-parser": { - "version": "1.44.0", - "resolved": "https://registry.npmjs.org/snyk-nodejs-lockfile-parser/-/snyk-nodejs-lockfile-parser-1.44.0.tgz", - "integrity": "sha512-A3vyLP2Vp0Zf14qXYiJsBL8wb+0RKdiBt5gpA1FcQDSePcC6UfLV3LsGVQucJIUE9dwB6t6RoeM34TL3+yEWdg==", + "version": "1.45.1", + "resolved": "https://registry.npmjs.org/snyk-nodejs-lockfile-parser/-/snyk-nodejs-lockfile-parser-1.45.1.tgz", + "integrity": "sha512-uj3dGoruveEZ1l/i5Qv4RS+dTBgAfmXXvTEmvH14fNuTEzDzb/PFe6+H1VPExrihwjrjHci1WIUhm9lQpYZZCQ==", "dependencies": { "@snyk/dep-graph": "^2.3.0", "@snyk/graphlib": "2.1.9-patch.3", @@ -33580,9 +33580,9 @@ } }, "snyk-nodejs-lockfile-parser": { - "version": "1.44.0", - "resolved": "https://registry.npmjs.org/snyk-nodejs-lockfile-parser/-/snyk-nodejs-lockfile-parser-1.44.0.tgz", - "integrity": "sha512-A3vyLP2Vp0Zf14qXYiJsBL8wb+0RKdiBt5gpA1FcQDSePcC6UfLV3LsGVQucJIUE9dwB6t6RoeM34TL3+yEWdg==", + "version": "1.45.1", + "resolved": "https://registry.npmjs.org/snyk-nodejs-lockfile-parser/-/snyk-nodejs-lockfile-parser-1.45.1.tgz", + "integrity": "sha512-uj3dGoruveEZ1l/i5Qv4RS+dTBgAfmXXvTEmvH14fNuTEzDzb/PFe6+H1VPExrihwjrjHci1WIUhm9lQpYZZCQ==", "requires": { "@snyk/dep-graph": "^2.3.0", "@snyk/graphlib": "2.1.9-patch.3", diff --git a/package.json b/package.json index 521785cfb7a..31dfe9b206a 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "snyk-gradle-plugin": "3.24.6", "snyk-module": "3.1.0", "snyk-mvn-plugin": "2.32.0", - "snyk-nodejs-lockfile-parser": "1.44.0", + "snyk-nodejs-lockfile-parser": "1.45.1", "snyk-nuget-plugin": "1.23.5", "snyk-php-plugin": "1.9.2", "snyk-policy": "^1.25.0", diff --git a/src/lib/plugins/nodejs-plugin/index.ts b/src/lib/plugins/nodejs-plugin/index.ts index a713269c6f9..7b45198b88f 100644 --- a/src/lib/plugins/nodejs-plugin/index.ts +++ b/src/lib/plugins/nodejs-plugin/index.ts @@ -4,6 +4,8 @@ import * as types from '../types'; import * as analytics from '../../analytics'; import { MissingTargetFileError } from '../../errors/missing-targetfile-error'; import { MultiProjectResult } from '@snyk/cli-interface/legacy/plugin'; +import { DepGraph } from '@snyk/dep-graph'; +import { PkgTree } from 'snyk-nodejs-lockfile-parser'; export async function inspect( root: string, @@ -18,12 +20,21 @@ export async function inspect( targetFile.endsWith('yarn.lock'); const getLockFileDeps = isLockFileBased && !options.traverseNodeModules; - const depTree: any = getLockFileDeps + const depRes: PkgTree | DepGraph = getLockFileDeps ? await lockParser.parse(root, targetFile, options) : await modulesParser.parse(root, targetFile, options); - if (depTree?.meta?.lockfileVersion) { - analytics.add('lockfileVersion', depTree.meta.lockfileVersion); + let scannedProjects: any[] = []; + if (isResDepGraph(depRes)) { + if (depRes.pkgManager.version) { + analytics.add('lockfileVersion', depRes?.pkgManager.version); + } + scannedProjects = [{ depGraph: depRes }]; + } else { + if (depRes.meta?.lockfileVersion) { + analytics.add('lockfileVersion', depRes?.meta?.lockfileVersion); + } + scannedProjects = [{ depTree: depRes }]; } return { @@ -31,6 +42,10 @@ export async function inspect( name: 'snyk-nodejs-lockfile-parser', runtime: process.version, }, - scannedProjects: [{ depTree }], + scannedProjects, }; } + +function isResDepGraph(depRes: PkgTree | DepGraph): depRes is DepGraph { + return 'rootPkg' in depRes; +} diff --git a/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts b/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts index b4cf8713321..7270c5819d8 100644 --- a/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts +++ b/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts @@ -5,14 +5,20 @@ import { spinner } from '../../spinner'; import * as analytics from '../../analytics'; import * as fs from 'fs'; import * as lockFileParser from 'snyk-nodejs-lockfile-parser'; -import { PkgTree } from 'snyk-nodejs-lockfile-parser'; +import { + NodeLockfileVersion, + PkgTree, + InvalidUserInputError, + ProjectParseOptions, +} from 'snyk-nodejs-lockfile-parser'; import { Options } from '../types'; +import { DepGraph } from '@snyk/dep-graph'; export async function parse( root: string, targetFile: string, options: Options, -): Promise { +): Promise { const lockFileFullPath = path.resolve(root, targetFile); if (!fs.existsSync(lockFileFullPath)) { throw new Error( @@ -47,9 +53,31 @@ export async function parse( }); const resolveModuleSpinnerLabel = `Analyzing npm dependencies for ${lockFileFullPath}`; debug(resolveModuleSpinnerLabel); + + const strictOutOfSync = options.strictOutOfSync !== false; + const lockfileVersion = lockFileParser.getLockfileVersionFromFile( + lockFileFullPath, + ); + if ( + lockfileVersion === NodeLockfileVersion.YarnLockV1 || + lockfileVersion === NodeLockfileVersion.YarnLockV2 + ) { + return await buildDepGraph( + root, + manifestFileFullPath, + lockFileFullPath, + lockfileVersion, + { + includeDevDeps: options.dev || false, + includeOptionalDeps: true, + strictOutOfSync, + pruneCycles: true, + }, + ); + } + try { await spinner(resolveModuleSpinnerLabel); - const strictOutOfSync = options.strictOutOfSync !== false; return lockFileParser.buildDepTreeFromFiles( root, manifestFileFullPath, @@ -61,3 +89,45 @@ export async function parse( await spinner.clear(resolveModuleSpinnerLabel)(); } } + +async function buildDepGraph( + root: string, + manifestFilePath: string, + lockfilePath: string, + lockfileVersion: NodeLockfileVersion, + options: ProjectParseOptions, +): Promise { + const manifestFileFullPath = path.resolve(root, manifestFilePath); + const lockFileFullPath = path.resolve(root, lockfilePath); + + if (!fs.existsSync(manifestFileFullPath)) { + throw new InvalidUserInputError( + 'Target file package.json not found at ' + + `location: ${manifestFileFullPath}`, + ); + } + if (!fs.existsSync(lockFileFullPath)) { + throw new InvalidUserInputError( + 'Lockfile not found at location: ' + lockFileFullPath, + ); + } + + const manifestFileContents = fs.readFileSync(manifestFileFullPath, 'utf-8'); + const lockFileContents = fs.readFileSync(lockFileFullPath, 'utf-8'); + + switch (lockfileVersion) { + case NodeLockfileVersion.YarnLockV1: + return await lockFileParser.parseYarnLockV1Project( + manifestFileContents, + lockFileContents, + options, + ); + case NodeLockfileVersion.YarnLockV2: + return lockFileParser.parseYarnLockV2Project( + manifestFileContents, + lockFileContents, + options, + ); + } + throw new Error('Failed to build dep graph from current project'); +} diff --git a/test/jest/acceptance/snyk-test/all-projects.spec.ts b/test/jest/acceptance/snyk-test/all-projects.spec.ts index 75a3cafb8b7..a7b292f74c7 100644 --- a/test/jest/acceptance/snyk-test/all-projects.spec.ts +++ b/test/jest/acceptance/snyk-test/all-projects.spec.ts @@ -56,7 +56,7 @@ describe('snyk test --all-projects (mocked server only)', () => { '✗ 1/3 potential projects failed to get dependencies', ); expect(stderr).toMatch( - `Dependency snyk was not found in yarn.lock. Your package.json and yarn.lock are probably out of sync. Please run "yarn install" and try again.`, + `Dependency snyk@1.320.0 was not found in yarn.lock. Your package.json and yarn.lock are probably out of sync. Please run "yarn install" and try again.`, ); }); @@ -81,7 +81,7 @@ describe('snyk test --all-projects (mocked server only)', () => { '✗ 1/3 potential projects failed to get dependencies', ); expect(stderr).toMatch( - `Dependency snyk was not found in yarn.lock. Your package.json and yarn.lock are probably out of sync. Please run "yarn install" and try again.`, + `Dependency snyk@1.320.0 was not found in yarn.lock. Your package.json and yarn.lock are probably out of sync. Please run "yarn install" and try again.`, ); }); diff --git a/test/tap/cli-monitor/cli-monitor.all-projects.spec.ts b/test/tap/cli-monitor/cli-monitor.all-projects.spec.ts index e8754b58a25..eab20e9bdb5 100644 --- a/test/tap/cli-monitor/cli-monitor.all-projects.spec.ts +++ b/test/tap/cli-monitor/cli-monitor.all-projects.spec.ts @@ -190,7 +190,7 @@ export const AllProjectsTests: AcceptanceTests = { ); t.match( result, - 'Dependency snyk was not found in yarn.lock', + 'Dependency snyk@* was not found in yarn.lock', 'yarn project had an error and we displayed it', ); diff --git a/test/tap/cli-test/cli-test.yarn.spec.ts b/test/tap/cli-test/cli-test.yarn.spec.ts index ba20ea78f2b..4daddddad37 100644 --- a/test/tap/cli-test/cli-test.yarn.spec.ts +++ b/test/tap/cli-test/cli-test.yarn.spec.ts @@ -15,7 +15,7 @@ export const YarnTests: AcceptanceTests = { t.equal( e.message, '\nTesting yarn-out-of-sync...\n\n' + - 'Dependency snyk was not found in yarn.lock.' + + 'Dependency snyk@* was not found in yarn.lock.' + ' Your package.json and yarn.lock are probably out of sync.' + ' Please run "yarn install" and try again.', 'Contains enough info about err',