From 9984e4ddfbbc08aff5547b0203fc2df06e71e940 Mon Sep 17 00:00:00 2001 From: Teodora Sandu Date: Thu, 29 Apr 2021 14:18:19 +0100 Subject: [PATCH 1/2] test: add smoke tests for verifying status code Co-authored-by: Ron Tal --- .../iac/no_vulnerabilities/pod-invalid.yaml | 2 + .../no_vulnerabilities/pod-privileged.yaml | 10 +++ .../spec/iac/snyk_test_local_exec_spec.sh | 78 ++++++++++++++----- 3 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 test/fixtures/iac/no_vulnerabilities/pod-invalid.yaml create mode 100644 test/fixtures/iac/no_vulnerabilities/pod-privileged.yaml diff --git a/test/fixtures/iac/no_vulnerabilities/pod-invalid.yaml b/test/fixtures/iac/no_vulnerabilities/pod-invalid.yaml new file mode 100644 index 00000000000..5a9619f82c8 --- /dev/null +++ b/test/fixtures/iac/no_vulnerabilities/pod-invalid.yaml @@ -0,0 +1,2 @@ +apiVersion: test +kind: example diff --git a/test/fixtures/iac/no_vulnerabilities/pod-privileged.yaml b/test/fixtures/iac/no_vulnerabilities/pod-privileged.yaml new file mode 100644 index 00000000000..2c1e0dbf2c8 --- /dev/null +++ b/test/fixtures/iac/no_vulnerabilities/pod-privileged.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: example +spec: + containers: + - name: example + image: example:latest + securityContext: + privileged: false diff --git a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh b/test/smoke/spec/iac/snyk_test_local_exec_spec.sh index 3c7f8758f30..9f028d33157 100644 --- a/test/smoke/spec/iac/snyk_test_local_exec_spec.sh +++ b/test/smoke/spec/iac/snyk_test_local_exec_spec.sh @@ -9,7 +9,7 @@ Describe "Snyk iac test --experimental command" Describe "basic usage" It "outputs an error if the --experimental flag is mistyped" When run snyk iac test ../fixtures/iac/kubernetes/pod-invalid.yaml --experimentl - The status should be failure + The status should equal 2 The output should include "Unsupported flag" End End @@ -20,7 +20,7 @@ Describe "Snyk iac test --experimental command" When run snyk iac test ../fixtures/iac/file-logging -d --experimental # We expect the output, specifically the analytics block not to include # the following text from the file. - The status should be failure # issues found + The status should equal 1 # issues found The output should not include "PRIVATE_FILE_CONTENT_CHECK" The error should not include "PRIVATE_FILE_CONTENT_CHECK" End @@ -29,7 +29,7 @@ Describe "Snyk iac test --experimental command" Describe "k8s single file scan" It "finds issues in k8s file" When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental - The status should be failure # issues found + The status should equal 1 # issues found The output should include "Testing pod-privileged.yaml..." # Outputs issues @@ -40,7 +40,7 @@ Describe "Snyk iac test --experimental command" It "filters out issues when using severity threshold" When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --severity-threshold=high - The status should be failure # one issue found + The status should equal 1 # one issue found The output should include "Testing pod-privileged.yaml..." The output should include "Infrastructure as code issues:" @@ -50,27 +50,27 @@ Describe "Snyk iac test --experimental command" It "outputs an error for files with no valid k8s objects" When run snyk iac test ../fixtures/iac/kubernetes/pod-invalid.yaml --experimental - The status should be failure + The status should equal 2 The output should include "We were unable to detect whether the YAML file" End It "outputs an error for Helm files" When run snyk iac test ../fixtures/iac/kubernetes/helm-config.yaml --experimental - The status should be failure + The status should equal 2 The output should include "We were unable to parse the YAML file" The output should include "do not support scanning of Helm files" End It "outputs the expected text when running with --sarif flag" When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --sarif - The status should be failure + The status should equal 1 The output should include '"id": "SNYK-CC-K8S-1",' The output should include '"ruleId": "SNYK-CC-K8S-1",' End It "outputs the expected text when running with --json flag" When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --json - The status should be failure + The status should equal 1 The output should include '"id": "SNYK-CC-K8S-1",' The output should include '"packageManager": "k8sconfig",' The result of function check_valid_json should be success @@ -80,7 +80,7 @@ Describe "Snyk iac test --experimental command" Describe "terraform single file scan" It "finds issues in terraform file" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental - The status should be failure # issues found + The status should equal 1 # issues found The output should include "Testing sg_open_ssh.tf..." # Outputs issues @@ -91,7 +91,7 @@ Describe "Snyk iac test --experimental command" It "filters out issues when using severity threshold" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --severity-threshold=high - The status should be success # no issues found + The status should equal 0 # no issues found The output should include "Testing sg_open_ssh.tf..." The output should include "Infrastructure as code issues:" @@ -102,20 +102,20 @@ Describe "Snyk iac test --experimental command" # will be fixed before beta It "outputs an error for invalid terraforom files" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh_invalid_hcl2.tf --experimental - The status should be failure + The status should equal 2 The output should include "We were unable to parse the Terraform file" End It "outputs the expected text when running with --sarif flag" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --sarif - The status should be failure + The status should equal 1 The output should include '"id": "SNYK-CC-TF-1",' The output should include '"ruleId": "SNYK-CC-TF-1",' End It "outputs the expected text when running with --json flag" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --json - The status should be failure + The status should equal 1 The output should include '"id": "SNYK-CC-TF-1",' The output should include '"packageManager": "terraformconfig",' The result of function check_valid_json should be success @@ -123,7 +123,7 @@ Describe "Snyk iac test --experimental command" It "outputs the expected text when running with --json flag and getting no vulnerabilities" When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --severity-threshold=high --json - The status should be success # no issues found + The status should equal 0 # no issues found The output should not include '"id": "SNYK-CC-TF-1",' The output should include '"packageManager": "terraformconfig",' The result of function check_valid_json should be success @@ -133,7 +133,7 @@ Describe "Snyk iac test --experimental command" Describe "directory scanning" It "finds issues in a directory with Terraform files" When run snyk iac test ../fixtures/iac/terraform/ --experimental - The status should be failure # issues found + The status should equal 1 # issues found # First File The output should include "Testing sg_open_ssh.tf..." The output should include "Infrastructure as code issues:" @@ -148,7 +148,7 @@ Describe "Snyk iac test --experimental command" It "finds issues in a directory with Kubernetes files" When run snyk iac test ../fixtures/iac/kubernetes/ --experimental - The status should be failure # issues found + The status should equal 1 # issues found # First File The output should include "Testing pod-privileged.yaml..." The output should include "Infrastructure as code issues:" @@ -163,7 +163,7 @@ Describe "Snyk iac test --experimental command" It "limits the depth of the directories" When run snyk iac test ../fixtures/iac/depth_detection/ --experimental --detection-depth=2 - The status should be success # no issues found + The status should equal 0 # no issues found # Only File The output should include "Testing one.tf..." The output should include "Infrastructure as code issues:" @@ -177,6 +177,46 @@ Describe "Snyk iac test --experimental command" # Directory scan summary The output should include "Tested 2 projects, no vulnerable paths were found." End + + Describe "Testing status code when issues found" + Describe "Using the --json flag" + It "returns 1 even if some files failed to parse" + When run snyk iac test ../fixtures/iac/kubernetes/ --experimental --json + The status should equal 1 + The output should not equal "" + The stderr should equal "" + End + End + + Describe "Not using the --json flag" + It "returns 1 even if some files failed to parse" + When run snyk iac test ../fixtures/iac/kubernetes/ --experimental + The status should equal 1 + The output should not equal "" + The stderr should equal "" + End + End + End + + Describe "Testing status code when no issues found" + Describe "Using the --json flag" + It "returns 0 even if some files failed to parse" + When run snyk iac test ../fixtures/iac/no_vulnerabilities/ --experimental --severity-threshold=high --json + The status should equal 0 + The output should not equal "" + The stderr should equal "" + End + End + + Describe "Not using the --json flag" + It "returns 0 even if some files failed to parse" + When run snyk iac test ../fixtures/iac/no_vulnerabilities/ --experimental --severity-threshold=high + The status should equal 0 + The output should not equal "" + The stderr should equal "" + End + End + End End Describe "Terraform plan scanning" @@ -184,7 +224,7 @@ Describe "Snyk iac test --experimental command" # in the future a flag will be added to control this functionality. It "finds issues in a Terraform plan file" When run snyk iac test ../fixtures/iac/terraform-plan/tf-plan.json --experimental - The status should be failure # issues found + The status should equal 1 # issues found The output should include "Testing tf-plan.json" # Outputs issues @@ -199,7 +239,7 @@ Describe "Snyk iac test --experimental command" # The test below should be enabled once we add the full scan flag xIt "finds issues in a Terraform plan file - full scan flag" When run snyk iac test ../fixtures/iac/terraform-plan/tf-plan.json --experimental - The status should be failure # issues found + The status should equal 1 # issues found The output should include "Testing tf-plan.json" # Outputs issues From 3048dcff3994c4aaad4c01e524c706289b04ee8c Mon Sep 17 00:00:00 2001 From: Teodora Sandu Date: Thu, 29 Apr 2021 16:59:04 +0100 Subject: [PATCH 2/2] fix: change how we check for success under the --json flag Co-authored-by: Ron Tal --- src/cli/commands/test/index.ts | 11 +---------- src/lib/snyk-test/iac-test-result.ts | 6 ++++-- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/cli/commands/test/index.ts b/src/cli/commands/test/index.ts index dc0a68889c6..c25537bd981 100644 --- a/src/cli/commands/test/index.ts +++ b/src/cli/commands/test/index.ts @@ -171,17 +171,8 @@ async function test(...args: MethodArgs): Promise { } = extractDataToSendFromResults(results, jsonData, options); if (options.json || options.sarif) { - // the new experimental IaC flow does not have the ok field because it returns vulnerabilities and errors separately - // the legacy IaC flow may return errors that get mapped into errorMappedResults, but that flow will be removed soon - const successfulIacScanning = - options.iac && - !foundVulnerabilities && - (!iacScanFailures || iacScanFailures.length === 0); - - // if running iac and it was successful, or // if all results are ok (.ok == true) - // then return the json - if (successfulIacScanning || errorMappedResults.every((res) => res.ok)) { + if (errorMappedResults.every((res) => res.ok)) { return TestCommandResult.createJsonTestCommandResult( stringifiedData, stringifiedJsonData, diff --git a/src/lib/snyk-test/iac-test-result.ts b/src/lib/snyk-test/iac-test-result.ts index 8056ae64647..8fd970e34de 100644 --- a/src/lib/snyk-test/iac-test-result.ts +++ b/src/lib/snyk-test/iac-test-result.ts @@ -49,6 +49,8 @@ export function mapIacTestResult( }; } + const infrastructureAsCodeIssues = + iacTest?.result?.cloudConfigResults.map(mapIacIssue) || []; const { result: { projectType }, ...filteredIacTest @@ -56,8 +58,8 @@ export function mapIacTestResult( return { ...filteredIacTest, projectType, - [IAC_ISSUES_KEY]: - iacTest?.result?.cloudConfigResults.map(mapIacIssue) || [], + ok: infrastructureAsCodeIssues.length === 0, + [IAC_ISSUES_KEY]: infrastructureAsCodeIssues, }; }