Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report ML-powered query enablement in the init status report #907

Merged
merged 10 commits into from Feb 7, 2022
8 changes: 5 additions & 3 deletions lib/config-utils.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/config-utils.js.map

Large diffs are not rendered by default.

32 changes: 15 additions & 17 deletions lib/config-utils.test.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/config-utils.test.js.map

Large diffs are not rendered by default.

19 changes: 13 additions & 6 deletions lib/util.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/util.js.map

Large diffs are not rendered by default.

11 changes: 9 additions & 2 deletions lib/util.test.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/util.test.js.map

Large diffs are not rendered by default.

77 changes: 56 additions & 21 deletions src/config-utils.test.ts
Expand Up @@ -1713,8 +1713,9 @@ const mlPoweredQueriesMacro = test.macro({
t: ExecutionContext,
codeQLVersion: string,
isMlPoweredQueriesFlagEnabled: boolean,
packsInput: string | undefined,
queriesInput: string | undefined,
shouldRunMlPoweredQueries: boolean
expectedVersionString: string | undefined
) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
Expand All @@ -1735,7 +1736,7 @@ const mlPoweredQueriesMacro = test.macro({
const { packs } = await configUtils.initConfig(
"javascript",
queriesInput,
undefined,
packsInput,
undefined,
undefined,
false,
Expand All @@ -1755,12 +1756,12 @@ const mlPoweredQueriesMacro = test.macro({
),
getRunnerLogger(true)
);
if (shouldRunMlPoweredQueries) {
if (expectedVersionString !== undefined) {
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "codeql/javascript-experimental-atm-queries",
version: "~0.0.2",
version: expectedVersionString,
},
],
});
Expand All @@ -1773,24 +1774,58 @@ const mlPoweredQueriesMacro = test.macro({
_providedTitle: string | undefined,
codeQLVersion: string,
isMlPoweredQueriesFlagEnabled: boolean,
packsInput: string | undefined,
queriesInput: string | undefined,
shouldRunMlPoweredQueries: boolean
) => {
const queriesInputDescription = queriesInput
? `'queries: ${queriesInput}'`
: "default config";

return `ML-powered queries ${
shouldRunMlPoweredQueries ? "are" : "aren't"
} loaded for ${queriesInputDescription} using CLI v${codeQLVersion} when feature flag is ${
expectedVersionString: string | undefined
) =>
`ML-powered queries ${
expectedVersionString !== undefined
? `${expectedVersionString} are`
: "aren't"
} loaded for packs: ${packsInput}, queries: ${queriesInput} using CLI v${codeQLVersion} when feature flag is ${
isMlPoweredQueriesFlagEnabled ? "enabled" : "disabled"
}`;
},
}`,
});

// macro, isMlPoweredQueriesFlagEnabled, queriesInput, shouldRunMlPoweredQueries
test(mlPoweredQueriesMacro, "2.7.4", true, "security-extended", false);
test(mlPoweredQueriesMacro, "2.7.5", false, "security-extended", false);
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, false);
test(mlPoweredQueriesMacro, "2.7.5", true, "security-extended", true);
test(mlPoweredQueriesMacro, "2.7.5", true, "security-and-quality", true);
// macro, isMlPoweredQueriesFlagEnabled, packsInput, queriesInput, versionString
test(
mlPoweredQueriesMacro,
"2.7.4",
true,
undefined,
"security-extended",
undefined
);
test(
mlPoweredQueriesMacro,
"2.7.5",
false,
undefined,
"security-extended",
undefined
);
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
undefined,
"security-extended",
"~0.0.2"
);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
undefined,
"security-and-quality",
"~0.0.2"
);
test(
mlPoweredQueriesMacro,
"2.7.5",
true,
"codeql/javascript-experimental-atm-queries@0.0.1",
"security-and-quality",
"0.0.1"
);
12 changes: 9 additions & 3 deletions src/config-utils.ts
Expand Up @@ -289,12 +289,18 @@ async function addBuiltinSuiteQueries(
throw new Error(getQueryUsesInvalid(configFile, suiteName));
}

// If we're running the JavaScript security-extended analysis (or a superset of it) and the repo
// is opted into the ML-powered queries beta, then add the ML-powered query pack so that we run
// the ML-powered queries.
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries.
if (
languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") &&
!(
packs.javascript &&
packs.javascript.some(
henrymercer marked this conversation as resolved.
Show resolved Hide resolved
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
)
) &&
(await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
(await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
) {
Expand Down
11 changes: 9 additions & 2 deletions src/util.test.ts
Expand Up @@ -297,8 +297,15 @@ const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
[[], "false"],
[[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
[
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" }],
"0.0.2",
[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.0.2" }],
"~0.0.2",
],
[
[
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
],
"multiple",
],
[
[
Expand Down
21 changes: 14 additions & 7 deletions src/util.ts
Expand Up @@ -631,24 +631,31 @@ export function checkNotWindows11() {
export const ML_POWERED_JS_QUERIES_PACK_NAME =
"codeql/javascript-experimental-atm-queries";

/** Get information about ML-powered JS queries to populate status reports with.
/**
* Get information about ML-powered JS queries to populate status reports with.
*
* This will be:
*
* - The version string if the analysis will use a specific version of the pack
* - "latest" if the analysis will use the latest version of the pack
* - "false" if the analysis won't run any ML-powered JS queries, or if the analysis will run
* multiple ML-powered JS query packs
* - "false" if the analysis won't run any ML-powered JS queries
* - "multiple" if the analysis will run multiple ML-powered JS query packs (this is an unsupported
* scenario)
*
* This function lives here rather than in `init-action.ts` so it's easier to test, since tests for
* `init-action.ts` would each need to live in their own file. See `analyze-action-env.ts` for an
* explanation as to why this is.
*/
export function getMlPoweredJsQueriesStatus(config: Config) {
export function getMlPoweredJsQueriesStatus(config: Config): string {
const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
(pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
);
return mlPoweredJsQueryPacks.length === 1
? mlPoweredJsQueryPacks[0].version || "latest"
: "false";
switch (mlPoweredJsQueryPacks.length) {
case 1:
return mlPoweredJsQueryPacks[0].version || "latest";
case 0:
return "false";
default:
return "multiple";
henrymercer marked this conversation as resolved.
Show resolved Hide resolved
}
}