Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: fix app-vuln-container-project acceptance tests instability #4868

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix: fix app-vuln-container-project acceptance tests instability #4868

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
232 changes: 133 additions & 99 deletions test/jest/acceptance/snyk-test/app-vuln-container-project.spec.ts
View file
Open in desktop
Expand Up @@ -3,105 +3,139 @@ import { fakeServer } from '../../../acceptance/fake-server';
import { runSnykCLI } from '../../util/runSnykCLI';

describe('container test projects behavior with --app-vulns, --file and --exclude-base-image-vulns flags', () => {
it('should find nothing when only vulns are in base image', async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-app-alpine-and-debug.tar --exclude-app-vulns --json --exclude-base-image-vulns`,
);

const jsonOutput = JSON.parse(stdout);
expect(jsonOutput.ok).toEqual(true);
expect(code).toEqual(0);
}, 30000);
it('should find all vulns including app vulns', async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --experimental`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput[0].ok).toEqual(false);
expect(jsonOutput[0].uniqueCount).toBeGreaterThan(0);
expect(jsonOutput[1].ok).toEqual(false);
expect(jsonOutput[1].uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
}, 30000);
it('should find nothing when app-vulns are explicitly disabled', async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --exclude-app-vulns`,
);
const jsonOutput = JSON.parse(stdout);
expect(Array.isArray(jsonOutput)).toBeFalsy();
expect(jsonOutput.applications).toBeUndefined();
expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
}, 30000);
it('should find nothing on conflicting app-vulns flags', async () => {
// if both flags are set, --exclude-app-vulns should take precedence and
// disable it.
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --app-vulns --exclude-app-vulns --experimental`,
);
const jsonOutput = JSON.parse(stdout);
expect(Array.isArray(jsonOutput)).toBeFalsy();
expect(jsonOutput.applications).toBeUndefined();
expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
}, 30000);

it('should show app vulns tip when available', async () => {
const { stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar`,
);

expect(stdout).toContain(`Testing docker-archive:test`);
}, 30000);

it('should find all vulns without experimental flag', async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
const applications = jsonOutput.applications;
expect(applications.length).toEqual(1);
expect(applications[0].uniqueCount).toBeGreaterThan(0);
expect(applications[0].ok).toEqual(false);
expect(code).toEqual(1);
}, 30000);
it('should return only dockerfile instructions vulnerabilities when excluding base image vulns', async () => {
const dockerfilePath = path.normalize(
'test/fixtures/container-projects/Dockerfile-vulns',
);

const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --exclude-app-vulns --json --file=${dockerfilePath} --exclude-base-image-vulns`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
}, 30000);

it('finds dockerfile instructions and app vulns when excluding base image vulns', async () => {
const dockerfilePath = path.normalize(
'test/fixtures/container-projects/Dockerfile-vulns',
);

const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --file=${dockerfilePath} --exclude-base-image-vulns`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(jsonOutput.applications[0].ok).toEqual(false);
expect(jsonOutput.applications[0].uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
}, 30000);
const TESTS_TIMEOUT_IN_MS = 60_000;

it(
'should find nothing when only vulns are in base image',
async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-app-alpine-and-debug.tar --exclude-app-vulns --json --exclude-base-image-vulns`,
);

const jsonOutput = JSON.parse(stdout);
expect(jsonOutput.ok).toEqual(true);
expect(code).toEqual(0);
},
TESTS_TIMEOUT_IN_MS,
);
it(
'should find all vulns including app vulns',
async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --experimental`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput[0].ok).toEqual(false);
expect(jsonOutput[0].uniqueCount).toBeGreaterThan(0);
expect(jsonOutput[1].ok).toEqual(false);
expect(jsonOutput[1].uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);
it(
'should find nothing when app-vulns are explicitly disabled',
async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --exclude-app-vulns`,
);
const jsonOutput = JSON.parse(stdout);
expect(Array.isArray(jsonOutput)).toBeFalsy();
expect(jsonOutput.applications).toBeUndefined();
expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);
it(
'should find nothing on conflicting app-vulns flags',
async () => {
// if both flags are set, --exclude-app-vulns should take precedence and
// disable it.
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --app-vulns --exclude-app-vulns --experimental`,
);
const jsonOutput = JSON.parse(stdout);
expect(Array.isArray(jsonOutput)).toBeFalsy();
expect(jsonOutput.applications).toBeUndefined();
expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);

it(
'should show app vulns tip when available',
async () => {
const { stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar`,
);

expect(stdout).toContain(`Testing docker-archive:test`);
},
TESTS_TIMEOUT_IN_MS,
);

it(
'should find all vulns without experimental flag',
async () => {
const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
const applications = jsonOutput.applications;
expect(applications.length).toEqual(1);
expect(applications[0].uniqueCount).toBeGreaterThan(0);
expect(applications[0].ok).toEqual(false);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);
it(
'should return only dockerfile instructions vulnerabilities when excluding base image vulns',
async () => {
const dockerfilePath = path.normalize(
'test/fixtures/container-projects/Dockerfile-vulns',
);

const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --exclude-app-vulns --json --file=${dockerfilePath} --exclude-base-image-vulns`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);

it(
'finds dockerfile instructions and app vulns when excluding base image vulns',
async () => {
const dockerfilePath = path.normalize(
'test/fixtures/container-projects/Dockerfile-vulns',
);

const { code, stdout } = await runSnykCLI(
`container test docker-archive:test/fixtures/container-projects/os-packages-and-app-vulns.tar --json --file=${dockerfilePath} --exclude-base-image-vulns`,
);
const jsonOutput = JSON.parse(stdout);

expect(jsonOutput.ok).toEqual(false);
expect(jsonOutput.uniqueCount).toBeGreaterThan(0);
expect(jsonOutput.applications[0].ok).toEqual(false);
expect(jsonOutput.applications[0].uniqueCount).toBeGreaterThan(0);
expect(code).toEqual(1);
},
TESTS_TIMEOUT_IN_MS,
);
});

describe('container test projects behavior with --json flag', () => {
Expand Down