Skip to content

Commit

Permalink
feat: [OSM-1040] pnpm support
Browse files Browse the repository at this point in the history
  • Loading branch information
@gemaxim
gemaxim committed Apr 18, 2024
1 parent 216af9c commit fb6735c
Show file tree
Hide file tree
Showing 93 changed files with 16,829 additions and 139 deletions.
502 changes: 487 additions & 15 deletions package-lock.json
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
View file
Expand Up @@ -121,7 +121,8 @@
"snyk-gradle-plugin": "4.1.0",
"snyk-module": "3.1.0",
"snyk-mvn-plugin": "3.3.1",
"snyk-nodejs-lockfile-parser": "1.52.11",
"snyk-nodejs-lockfile-parser": "1.53.1",
"snyk-nodejs-plugin": "1.0.1",
"snyk-nuget-plugin": "2.4.1",
"snyk-php-plugin": "1.9.2",
"snyk-policy": "^1.25.0",
Expand Down
1 change: 1 addition & 0 deletions src/cli/args.ts
View file
Expand Up @@ -44,6 +44,7 @@ const DEBUG_DEFAULT_NAMESPACES = [
'snyk-sbt-plugin',
'snyk-mvn-plugin',
'snyk-yarn-workspaces',
'snyk-pnpm-workspaces',
];

function dashToCamelCase(dash) {
Expand Down
27 changes: 21 additions & 6 deletions src/cli/commands/monitor/index.ts
View file
Expand Up @@ -49,6 +49,7 @@ import { getEcosystem, monitorEcosystem } from '../../../lib/ecosystems';
import { getFormattedMonitorOutput } from '../../../lib/ecosystems/monitor';
import { processCommandArgs } from '../process-command-args';
import { hasFeatureFlag } from '../../../lib/feature-flags';
import { PNPM_FEATURE_FLAG } from '../../../lib/package-managers';

const SEPARATOR = '\n-------------------------------------------------------\n';
const debug = Debug('snyk');
Expand Down Expand Up @@ -158,6 +159,12 @@ export default async function monitor(...args0: MethodArgs): Promise<any> {
);
}

const hasPnpmSupport = await hasFeatureFlag(PNPM_FEATURE_FLAG, options);

const featureFlags = hasPnpmSupport
? new Set<string>([PNPM_FEATURE_FLAG])
: new Set<string>();

// Part 1: every argument is a scan target; process them sequentially
for (const path of paths) {
debug(`Processing ${path}...`);
Expand All @@ -170,7 +177,11 @@ export default async function monitor(...args0: MethodArgs): Promise<any> {
} else if (options.docker) {
analysisType = 'docker';
} else {
packageManager = detect.detectPackageManager(path, options);
packageManager = detect.detectPackageManager(
path,
options,
featureFlags,
);
}
const unsupportedPackageManagers: Array<{
label: string;
Expand All @@ -185,7 +196,7 @@ export default async function monitor(...args0: MethodArgs): Promise<any> {
const targetFile =
!options.scanAllUnmanaged && options.docker && !options.file // snyk monitor --docker (without --file)
? undefined
: options.file || detect.detectPackageFile(path);
: options.file || detect.detectPackageFile(path, featureFlags);

const displayPath = pathUtil.relative(
'.',
Expand All @@ -206,11 +217,15 @@ export default async function monitor(...args0: MethodArgs): Promise<any> {
// each plugin will be asked to scan once per path
// some return single InspectResult & newer ones return Multi
const inspectResult = await promiseOrCleanup(
getDepsFromPlugin(path, {
...options,
getDepsFromPlugin(
path,
packageManager,
}),
{
...options,
path,
packageManager,
},
featureFlags,
),
spinner.clear(analyzingDepsSpinnerLabel),
);
analytics.add('pluginName', inspectResult.plugin.name);
Expand Down
84 changes: 75 additions & 9 deletions src/lib/detect.ts
View file
Expand Up @@ -5,6 +5,9 @@ import { NoSupportedManifestsFoundError } from './errors';
import {
SupportedPackageManagers,
SUPPORTED_MANIFEST_FILES,
SUPPORTED_MANIFEST_FILES_UNDER_FF,
SupportedPackageManagersUnderFeatureFlag,
PACKAGE_MANAGERS_FEATURE_FLAGS_MAP,
} from './package-managers';

const debug = debugLib('snyk-detect');
Expand Down Expand Up @@ -65,6 +68,10 @@ export const AUTO_DETECTABLE_FILES: string[] = [
'Package.swift',
];

export const AUTO_DETECTABLE_FILES_UNDER_FF = {
pnpm: 'pnpm-lock.yaml',
};

// when file is specified with --file, we look it up here
// this is also used when --all-projects flag is enabled and auto detection plugin is triggered
const DETECTABLE_PACKAGE_MANAGERS: {
Expand Down Expand Up @@ -102,7 +109,29 @@ const DETECTABLE_PACKAGE_MANAGERS: {
[SUPPORTED_MANIFEST_FILES.PACKAGE_SWIFT]: 'swift',
};

export function isPathToPackageFile(path: string) {
export const DETECTABLE_PACKAGE_MANAGERS_UNDER_FF: {
[key in SUPPORTED_MANIFEST_FILES_UNDER_FF]: SupportedPackageManagersUnderFeatureFlag;
} = {
[SUPPORTED_MANIFEST_FILES_UNDER_FF.PNPM_LOCK]: 'pnpm',
};

export function isPathToPackageFile(
path: string,
featureFlags: Set<string> = new Set<string>(),
) {
for (const fileName of Object.keys(DETECTABLE_PACKAGE_MANAGERS)) {
if (
path.endsWith(fileName) &&
featureFlags.has(
PACKAGE_MANAGERS_FEATURE_FLAGS_MAP[
DETECTABLE_PACKAGE_MANAGERS_UNDER_FF[fileName]
],
)
) {
return true;
}
}

for (const fileName of DETECTABLE_FILES) {
if (path.endsWith(fileName)) {
return true;
Expand All @@ -111,7 +140,11 @@ export function isPathToPackageFile(path: string) {
return false;
}

export function detectPackageManager(root: string, options) {
export function detectPackageManager(
root: string,
options,
featureFlags: Set<string> = new Set<string>(),
) {
// If user specified a package manager let's use it.
if (options.packageManager) {
return options.packageManager;
Expand All @@ -133,14 +166,14 @@ export function detectPackageManager(root: string, options) {
);
}
file = options.file;
packageManager = detectPackageManagerFromFile(file);
packageManager = detectPackageManagerFromFile(file, featureFlags);
} else if (options.scanAllUnmanaged) {
packageManager = 'maven';
} else {
debug('no file specified. Trying to autodetect in base folder ' + root);
file = detectPackageFile(root);
file = detectPackageFile(root, featureFlags);
if (file) {
packageManager = detectPackageManagerFromFile(file);
packageManager = detectPackageManagerFromFile(file, featureFlags);
}
}
} else {
Expand Down Expand Up @@ -169,20 +202,36 @@ export function isLocalFolder(root: string) {
}
}

export function detectPackageFile(root) {
export function detectPackageFile(
root: string,
featureFlags: Set<string> = new Set<string>(),
) {
for (const file of Object.keys(DETECTABLE_PACKAGE_MANAGERS_UNDER_FF)) {
if (
fs.existsSync(pathLib.resolve(root, file)) &&
featureFlags.has(
PACKAGE_MANAGERS_FEATURE_FLAGS_MAP[
DETECTABLE_PACKAGE_MANAGERS_UNDER_FF[file]
],
)
) {
return file;
}
}

for (const file of DETECTABLE_FILES) {
if (fs.existsSync(pathLib.resolve(root, file))) {
debug('found package file ' + file + ' in ' + root);
return file;
}
}

debug('no package file found in ' + root);
}

export function detectPackageManagerFromFile(
file: string,
): SupportedPackageManagers {
featureFlags: Set<string> = new Set<string>(),
): SupportedPackageManagers | SupportedPackageManagersUnderFeatureFlag {
let key = pathLib.basename(file);

// TODO: fix this to use glob matching instead
Expand All @@ -199,11 +248,28 @@ export function detectPackageManagerFromFile(
key = '.war';
}

if (!(key in DETECTABLE_PACKAGE_MANAGERS)) {
if (
!(key in DETECTABLE_PACKAGE_MANAGERS) &&
!(key in DETECTABLE_PACKAGE_MANAGERS_UNDER_FF)
) {
// we throw and error here because the file was specified by the user
throw new Error('Could not detect package manager for file: ' + file);
}

if (key in DETECTABLE_PACKAGE_MANAGERS_UNDER_FF) {
if (
featureFlags.has(
PACKAGE_MANAGERS_FEATURE_FLAGS_MAP[
DETECTABLE_PACKAGE_MANAGERS_UNDER_FF[key]
],
)
) {
return DETECTABLE_PACKAGE_MANAGERS_UNDER_FF[key];
} else {
throw new Error('Could not detect package manager for file: ' + file);
}
}

return DETECTABLE_PACKAGE_MANAGERS[key];
}

Expand Down
40 changes: 32 additions & 8 deletions src/lib/find-files.ts
View file
Expand Up @@ -5,6 +5,7 @@ const sortBy = require('lodash.sortby');
const groupBy = require('lodash.groupby');
import { detectPackageManagerFromFile } from './detect';
import * as debugModule from 'debug';
import { PNPM_FEATURE_FLAG } from './package-managers';

const debug = debugModule('snyk:find-files');

Expand Down Expand Up @@ -61,6 +62,7 @@ export async function find(
path: string,
ignore: string[] = [],
filter: string[] = [],
featureFlags: Set<string> = new Set<string>(),
levelsDeep = 4,
): Promise<FindFilesRes> {
const found: string[] = [];
Expand Down Expand Up @@ -90,6 +92,7 @@ export async function find(
path,
ignore,
filter,
featureFlags,
levelsDeep,
);
found.push(...files);
Expand All @@ -109,7 +112,10 @@ export async function find(
} files: ${filteredOutFiles.join(', ')}`,
);
}
return { files: filterForDefaultManifests(found), allFilesFound: foundAll };
return {
files: filterForDefaultManifests(found, featureFlags),
allFilesFound: foundAll,
};
} catch (err) {
throw new Error(`Error finding files in path '${path}'.\n${err.message}`);
}
Expand All @@ -131,6 +137,7 @@ async function findInDirectory(
path: string,
ignore: string[] = [],
filter: string[] = [],
featureFlags: Set<string> = new Set<string>(),
levelsDeep = 4,
): Promise<FindFilesRes> {
const files = await readDirectory(path);
Expand All @@ -142,7 +149,7 @@ async function findInDirectory(
debug('File does not seem to exist, skipping: ', file);
return { files: [], allFilesFound: [] };
}
return find(resolvedPath, ignore, filter, levelsDeep);
return find(resolvedPath, ignore, filter, featureFlags, levelsDeep);
});

const found = await Promise.all(toFind);
Expand All @@ -158,7 +165,10 @@ async function findInDirectory(
};
}

function filterForDefaultManifests(files: string[]): string[] {
function filterForDefaultManifests(
files: string[],
featureFlags: Set<string> = new Set<string>(),
): string[] {
// take all the files in the same dir & filter out
// based on package Manager
if (files.length <= 1) {
Expand All @@ -173,7 +183,7 @@ function filterForDefaultManifests(files: string[]): string[] {
.map((p) => ({
path: p,
...pathLib.parse(p),
packageManager: detectProjectTypeFromFile(p),
packageManager: detectProjectTypeFromFile(p, featureFlags),
}));
const sorted = sortBy(beforeSort, 'dir');
const foundFiles = groupBy(sorted, 'dir');
Expand Down Expand Up @@ -202,11 +212,12 @@ function filterForDefaultManifests(files: string[]): string[] {
const defaultManifestFileName = chooseBestManifest(
filesPerPackageManager,
packageManager,
featureFlags,
);
if (defaultManifestFileName) {
const shouldSkip = shouldSkipAddingFile(
packageManager,
filesPerPackageManager[0].path,
filesPerPackageManager[0].path, // defaultManifestFileName?
filteredFiles,
);
if (shouldSkip) {
Expand All @@ -219,12 +230,18 @@ function filterForDefaultManifests(files: string[]): string[] {
return filteredFiles;
}

function detectProjectTypeFromFile(file: string): string | null {
function detectProjectTypeFromFile(
file: string,
featureFlags: Set<string> = new Set<string>(),
): string | null {
try {
const packageManager = detectPackageManagerFromFile(file);
const packageManager = detectPackageManagerFromFile(file, featureFlags);
if (['yarn', 'npm'].includes(packageManager)) {
return 'node';
}
if (featureFlags.has(PNPM_FEATURE_FLAG) && packageManager === 'pnpm') {
return 'node';
}
return packageManager;
} catch (error) {
return null;
Expand Down Expand Up @@ -256,11 +273,18 @@ function shouldSkipAddingFile(
function chooseBestManifest(
files: Array<{ base: string; path: string }>,
projectType: string,
featureFlags: Set<string> = new Set<string>([]),
): string | null {
switch (projectType) {
case 'node': {
const nodeLockfiles = ['package-lock.json', 'yarn.lock'];
if (featureFlags.has(PNPM_FEATURE_FLAG)) {
nodeLockfiles.push('pnpm-lock.yaml');
}
const lockFile = files.filter((path) =>
['package-lock.json', 'yarn.lock'].includes(path.base),
['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'].includes(
path.base,
),
)[0];
debug(
`Encountered multiple node lockfiles files, defaulting to ${lockFile.path}`,
Expand Down
1 change: 1 addition & 0 deletions src/lib/formatters/open-source-sarif-output.ts
View file
Expand Up @@ -10,6 +10,7 @@ const LOCK_FILES_TO_MANIFEST_MAP = {
'Gemfile.lock': 'Gemfile',
'package-lock.json': 'package.json',
'yarn.lock': 'package.json',
'pnpm-lock.yaml': 'package.json',
'Gopkg.lock': 'Gopkg.toml',
'go.sum': 'go.mod',
'composer.lock': 'composer.json',
Expand Down

0 comments on commit fb6735c

Please sign in to comment.