Merge pull request #1870 from snyk/fix/json-status-code-iac

fix: consistent command termination between --json and no --json flag

src/cli/commands/test/index.ts

@@ -171,17 +171,8 @@ async function test(...args: MethodArgs): Promise<TestCommandResult> {
   } = extractDataToSendFromResults(results, jsonData, options);
 
   if (options.json || options.sarif) {
-    // the new experimental IaC flow does not have the ok field because it returns vulnerabilities and errors separately
-    // the legacy IaC flow may return errors that get mapped into errorMappedResults, but that flow will be removed soon
-    const successfulIacScanning =
-      options.iac &&
-      !foundVulnerabilities &&
-      (!iacScanFailures || iacScanFailures.length === 0);
-
-    // if running iac and it was successful, or
     // if all results are ok (.ok == true)
-    // then return the json
-    if (successfulIacScanning || errorMappedResults.every((res) => res.ok)) {
+    if (errorMappedResults.every((res) => res.ok)) {
       return TestCommandResult.createJsonTestCommandResult(
         stringifiedData,
         stringifiedJsonData,

src/lib/snyk-test/iac-test-result.ts

@@ -49,15 +49,17 @@ export function mapIacTestResult(
     };
   }
 
+  const infrastructureAsCodeIssues =
+    iacTest?.result?.cloudConfigResults.map(mapIacIssue) || [];
   const {
     result: { projectType },
     ...filteredIacTest
   } = iacTest;
   return {
     ...filteredIacTest,
     projectType,
-    [IAC_ISSUES_KEY]:
-      iacTest?.result?.cloudConfigResults.map(mapIacIssue) || [],
+    ok: infrastructureAsCodeIssues.length === 0,
+    [IAC_ISSUES_KEY]: infrastructureAsCodeIssues,
   };
 }
 

test/fixtures/iac/no_vulnerabilities/pod-invalid.yaml

@@ -0,0 +1,2 @@
+apiVersion: test
+kind: example

test/fixtures/iac/no_vulnerabilities/pod-privileged.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: example
+spec:
+  containers:
+    - name: example
+      image: example:latest
+      securityContext:
+        privileged: false 

test/smoke/spec/iac/snyk_test_local_exec_spec.sh

@@ -9,7 +9,7 @@ Describe "Snyk iac test --experimental command"
   Describe "basic usage"
     It "outputs an error if the --experimental flag is mistyped"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-invalid.yaml --experimentl
-      The status should be failure
+      The status should equal 2
       The output should include "Unsupported flag"
     End
   End
@@ -20,7 +20,7 @@ Describe "Snyk iac test --experimental command"
       When run snyk iac test ../fixtures/iac/file-logging -d --experimental
       # We expect the output, specifically the analytics block not to include
       # the following text from the file.
-      The status should be failure  # issues found
+      The status should equal 1 # issues found
       The output should not include "PRIVATE_FILE_CONTENT_CHECK"
       The error should not include "PRIVATE_FILE_CONTENT_CHECK"
     End
@@ -29,7 +29,7 @@ Describe "Snyk iac test --experimental command"
   Describe "k8s single file scan"
     It "finds issues in k8s file"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       The output should include "Testing pod-privileged.yaml..."
 
       # Outputs issues
@@ -40,7 +40,7 @@ Describe "Snyk iac test --experimental command"
 
     It "filters out issues when using severity threshold"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --severity-threshold=high
-      The status should be failure # one issue found
+      The status should equal 1 # one issue found
       The output should include "Testing pod-privileged.yaml..."
 
       The output should include "Infrastructure as code issues:"
@@ -50,27 +50,27 @@ Describe "Snyk iac test --experimental command"
 
     It "outputs an error for files with no valid k8s objects"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-invalid.yaml --experimental
-      The status should be failure
+      The status should equal 2
       The output should include "We were unable to detect whether the YAML file"
     End
 
     It "outputs an error for Helm files"
       When run snyk iac test ../fixtures/iac/kubernetes/helm-config.yaml --experimental
-      The status should be failure
+      The status should equal 2
       The output should include "We were unable to parse the YAML file"
       The output should include "do not support scanning of Helm files"
     End
 
     It "outputs the expected text when running with --sarif flag"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --sarif
-      The status should be failure
+      The status should equal 1
       The output should include '"id": "SNYK-CC-K8S-1",'
       The output should include '"ruleId": "SNYK-CC-K8S-1",'
     End
 
     It "outputs the expected text when running with --json flag"
       When run snyk iac test ../fixtures/iac/kubernetes/pod-privileged.yaml --experimental --json
-      The status should be failure
+      The status should equal 1
       The output should include '"id": "SNYK-CC-K8S-1",'
       The output should include '"packageManager": "k8sconfig",'
       The result of function check_valid_json should be success
@@ -80,7 +80,7 @@ Describe "Snyk iac test --experimental command"
   Describe "terraform single file scan"
     It "finds issues in terraform file"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       The output should include "Testing sg_open_ssh.tf..."
 
       # Outputs issues
@@ -91,7 +91,7 @@ Describe "Snyk iac test --experimental command"
 
     It "filters out issues when using severity threshold"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --severity-threshold=high
-      The status should be success # no issues found
+      The status should equal 0 # no issues found
       The output should include "Testing sg_open_ssh.tf..."
 
       The output should include "Infrastructure as code issues:"
@@ -102,28 +102,28 @@ Describe "Snyk iac test --experimental command"
     # will be fixed before beta
     It "outputs an error for invalid terraforom files"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh_invalid_hcl2.tf --experimental
-      The status should be failure
+      The status should equal 2
       The output should include "We were unable to parse the Terraform file"
     End
 
     It "outputs the expected text when running with --sarif flag"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --sarif
-      The status should be failure
+      The status should equal 1
       The output should include '"id": "SNYK-CC-TF-1",'
       The output should include '"ruleId": "SNYK-CC-TF-1",'
     End
 
     It "outputs the expected text when running with --json flag"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --json
-      The status should be failure
+      The status should equal 1
       The output should include '"id": "SNYK-CC-TF-1",'
       The output should include '"packageManager": "terraformconfig",'
       The result of function check_valid_json should be success
     End
 
     It "outputs the expected text when running with --json flag and getting no vulnerabilities"
       When run snyk iac test ../fixtures/iac/terraform/sg_open_ssh.tf --experimental --severity-threshold=high --json
-      The status should be success # no issues found
+      The status should equal 0 # no issues found
       The output should not include '"id": "SNYK-CC-TF-1",'
       The output should include '"packageManager": "terraformconfig",'
       The result of function check_valid_json should be success
@@ -133,7 +133,7 @@ Describe "Snyk iac test --experimental command"
   Describe "directory scanning"
     It "finds issues in a directory with Terraform files"
       When run snyk iac test ../fixtures/iac/terraform/ --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       # First File
       The output should include "Testing sg_open_ssh.tf..."
       The output should include "Infrastructure as code issues:"
@@ -148,7 +148,7 @@ Describe "Snyk iac test --experimental command"
 
     It "finds issues in a directory with Kubernetes files"
       When run snyk iac test ../fixtures/iac/kubernetes/ --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       # First File
       The output should include "Testing pod-privileged.yaml..."
       The output should include "Infrastructure as code issues:"
@@ -163,7 +163,7 @@ Describe "Snyk iac test --experimental command"
 
     It "limits the depth of the directories"
       When run snyk iac test ../fixtures/iac/depth_detection/ --experimental --detection-depth=2
-      The status should be success # no issues found
+      The status should equal 0 # no issues found
       # Only File
       The output should include "Testing one.tf..."
       The output should include "Infrastructure as code issues:"
@@ -177,14 +177,54 @@ Describe "Snyk iac test --experimental command"
       # Directory scan summary
       The output should include "Tested 2 projects, no vulnerable paths were found."
     End
+
+    Describe "Testing status code when issues found"
+      Describe "Using the --json flag"
+        It "returns 1 even if some files failed to parse"
+          When run snyk iac test ../fixtures/iac/kubernetes/ --experimental --json
+          The status should equal 1
+          The output should not equal ""
+          The stderr should equal ""
+        End
+      End
+
+      Describe "Not using the --json flag"
+        It "returns 1 even if some files failed to parse"
+          When run snyk iac test ../fixtures/iac/kubernetes/ --experimental
+          The status should equal 1
+          The output should not equal ""
+          The stderr should equal ""
+        End
+      End
+    End
+
+    Describe "Testing status code when no issues found"
+      Describe "Using the --json flag"
+        It "returns 0 even if some files failed to parse"
+          When run snyk iac test ../fixtures/iac/no_vulnerabilities/ --experimental --severity-threshold=high --json
+          The status should equal 0
+          The output should not equal ""
+          The stderr should equal ""
+        End
+      End
+
+      Describe "Not using the --json flag"
+        It "returns 0 even if some files failed to parse"
+          When run snyk iac test ../fixtures/iac/no_vulnerabilities/ --experimental --severity-threshold=high
+          The status should equal 0
+          The output should not equal ""
+          The stderr should equal ""
+        End
+      End
+    End
   End
 
   Describe "Terraform plan scanning"
     # Note that this now defaults to the delta scan, not the full scan.
     # in the future a flag will be added to control this functionality.
     It "finds issues in a Terraform plan file"
       When run snyk iac test ../fixtures/iac/terraform-plan/tf-plan.json --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       The output should include "Testing tf-plan.json"
 
       # Outputs issues
@@ -199,7 +239,7 @@ Describe "Snyk iac test --experimental command"
     # The test below should be enabled once we add the full scan flag
     xIt "finds issues in a Terraform plan file - full scan flag"
       When run snyk iac test ../fixtures/iac/terraform-plan/tf-plan.json --experimental
-      The status should be failure # issues found
+      The status should equal 1 # issues found
       The output should include "Testing tf-plan.json"
 
       # Outputs issues