Merge pull request #3838 from snyk/chore/create-iac-e2e-tests-cfg-2106

chore: Create smoke tests for experimental IaC tests

.github/CODEOWNERS

@@ -34,6 +34,7 @@ test/fixtures/container-app-vulns/ @snyk/mycelium
 test/fixtures/container-projects/ @snyk/mycelium @snyk/potion
 test/fixtures/docker/ @snyk/mycelium @snyk/potion
 test/fixtures/iac/ @snyk/group-infrastructure-as-code
+test/smoke/iac/ @snyk/group-infrastructure-as-code
 test/smoke/spec/iac/ @snyk/group-infrastructure-as-code
 test/smoke/spec/snyk_code_spec.sh @snyk/zenith
 test/smoke/spec/snyk_basic_spec.sh @snyk/hammer
@@ -70,6 +71,8 @@ test/fixtures/unmanaged-log4j-fixture @snyk/tundra
 test/jest/acceptance/snyk-log4shell/log4shell-detection.spec.ts @snyk/tundra
 test/jest/acceptance/snyk-test/app-vuln-container-project.spec.ts @snyk/mycelium
 /.github @snyk/hammer
+/.github/workflows/iac-smoke-tests.yml @snyk/group-infrastructure-as-code
+/.github/workflows/iac-smoke-tests-pulls.yml @snyk/group-infrastructure-as-code
 
 # tap tests ownership
 test/tap/cli-monitor/ @snyk/snyk-open-source

.github/workflows/iac-smoke-tests-pulls.yml

@@ -0,0 +1,92 @@
+name: Infrastructure as Code Smoke Tests (Pull Requests)
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  check_for_changed_iac_files:
+    name: Check for changed IaC files
+    runs-on: ubuntu-latest
+    outputs:
+      is_changed: ${{ steps.check_iac_files_changed.outputs.is_changed }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install jq
+        run: |
+          sudo apt-get install jq
+
+      - name: Parse CODEOWNERS file
+        id: codeowners
+        uses: SvanBoxel/codeowners-action@v2.2
+        with:
+          file_match_info: 'true'
+          path: ./.github/CODEOWNERS
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v29.0.4
+
+      - name: Get all IaC files
+        id: get_all_iac_files
+        run: |
+          ALL_IAC_FILES=$(
+            echo ${{ toJSON(steps.codeowners.outputs.filematches) }} |
+            jq '[
+              to_entries[] |
+              select(
+                .value.owners |
+                index("@snyk/group-infrastructure-as-code")
+              ) |
+              .key
+            ]'
+          )
+
+          echo "::set-output name=all_iac_files::$(
+            echo $ALL_IAC_FILES
+          )"
+
+      - id: check_iac_files_changed
+        name: Check for changed files owned by IaC
+        run: |
+          ALL_IAC_FILES=${{ toJson(steps.get_all_iac_files.outputs.all_iac_files) }}
+
+          CHANGED_FILES=$(
+            echo ${{ steps.changed-files.outputs.all_changed_files }} |
+            jq -R 'split(" ")'
+          )
+
+          CHANGED_IAC_FILES=$(
+            echo $CHANGED_FILES |
+            jq --argjson ALL_IAC_FILES "$ALL_IAC_FILES" '[
+                .[] |
+                . as $changed_file |
+                select(
+                  $ALL_IAC_FILES |
+                  index($changed_file)
+                )
+              ]'
+          )
+
+          CHANGED_IAC_FILES_COUNT=$(
+            echo $CHANGED_IAC_FILES | jq 'length'
+          )
+
+          IS_CHANGED=$(
+            echo $CHANGED_IAC_FILES_COUNT | jq '. > 0'
+          )
+
+          $IS_CHANGED &&
+          echo "Found $CHANGED_IAC_FILES_COUNT changed IaC files: $CHANGED_IAC_FILES"||
+          echo "No changed IaC files found!"
+
+          echo "::set-output name=is_changed::$IS_CHANGED"
+
+  run_iac_smoke_tests:
+    name: Run IaC smoke tests
+    uses: ./.github/workflows/iac-smoke-tests.yml
+    needs: check_for_changed_iac_files
+    if: ${{ needs.check_for_changed_iac_files.outputs.is_changed == 'true' }}
+    secrets: inherit

.github/workflows/iac-smoke-tests.yml

@@ -0,0 +1,67 @@
+name: Infrastructure as Code Smoke Tests
+
+on:
+  schedule:
+    - cron: '0 * * * *'
+  release:
+    types: [published]
+  workflow_call:
+
+jobs:
+  run_iac_e2e_tests:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 15
+
+      - name: Install jq on macOS
+        if: ${{ matrix.os == 'macos' }}
+        run: |
+          brew install jq
+
+      - name: Install jq on Windows
+        if: ${{ matrix.os == 'windows'}}
+        run: |
+          iwr -useb get.scoop.sh -outfile 'install-scoop.ps1'
+          .\install-scoop.ps1 -RunAsAdmin
+          scoop install jq
+
+      - name: Install jq on Ubuntu
+        if: ${{ matrix.os == 'ubuntu' }}
+        run: |
+          sudo apt-get install jq
+
+      - name: Install dependencies
+        run: |
+          npm install
+
+      - name: Build Snyk CLI
+        run: |
+          npm run build
+
+      - name: Run IaC smoke tests - non-Windows
+        if: ${{ matrix.os != 'windows' }}
+        env:
+          IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
+          TEST_SNYK_COMMAND: ${{ format('node {0}/dist/cli/index.js', github.workspace) }}
+        run: |
+          npm run test:smoke:iac
+
+      - name: Run IaC smoke tests - Windows
+        if: ${{ matrix.os == 'windows' }}
+        shell: pwsh
+        env:
+          IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
+          TEST_SNYK_COMMAND: ${{ format('node {0}\dist\cli\index.js', github.workspace) }}
+        run: |
+          npm run test:smoke:iac

package.json

@@ -43,7 +43,8 @@
     "test:unit": "jest --runInBand --testPathPattern '/test(/jest)?/unit/'",
     "test:acceptance": "jest --runInBand --testPathPattern '/test(/jest)?/acceptance/'",
     "test:tap": "tap -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register test/tap/*.test.* ",
-    "test:smoke": "./scripts/run-smoke-tests-locally.sh"
+    "test:smoke": "./scripts/run-smoke-tests-locally.sh",
+    "test:smoke:iac": "jest --runInBand --testPathPattern '/test/smoke(/jest)?/iac/'"
   },
   "keywords": [
     "security",

test/smoke/iac/README.md

@@ -0,0 +1,39 @@
+# Snyk Infrastructure as Code Smoke Tests
+
+Design goal is to have a single test suite, aligned with the scope of the Snyk CLI's smoke tests, that can detect if IaC commands do not work properly - before and after it's released. The tests help us incorporate resources and requests made via network calls, to provide better coverage for the end to end flow of these commands. Some examples:
+
+- Network calls made for fetching org properties, such as feature flags data, custom severities, etc.
+- Downloading resources from CDNs, e.g., binary executables, ruleset bundles, etc.
+
+The tests were written with Jest, and use a Snyk CLI executable either configured in the PATH environment variable, or overrode, using the `TEST_SNYK_COMMAND` environment variable, see more in the 'Notes on the
+
+# Implementation details and usage
+
+These smoke tests are written with Jest, using the Snyk CLI executable identified on the runtime environment (See 'Notes on the local run' section below to read on how to override it)
+
+Spec in this folder is used as a
+
+1. **"name: Infrastructure as Code Smoke Tests" Github Action** - these run every hour and upon releases.
+2. **["Infrastructure as Code Smoke Tests (Pull Requests)"] GitHub Action** - these run for pull requests to the `master` branch which include changes to files owned by group IaC.
+
+```sh
+npm run test:smoke:iac
+```
+
+### Notes on the local run
+
+These tests can be executed with the following npm script:
+
+```
+npm run test:smoke:iac
+```
+
+Alternatively, they can be executed directly via `jest`, by running:
+
+```
+npx jest test/smoke/iac/
+```
+
+You may specify any executable that will be used by the smoke tests, by configuring the `TEST_SNYK_COMMAND` environment variable. E.g. a local exuctable `TEST_SNYK_COMMAND="./snyk-macos"` or an `TEST_SNYK_COMMAND="npx snyk@1.500.0"` or `TEST_SNYK_COMMAND="node ./dist/cli"` for local execution.
+
+You may also configure an authentication token with the `SNYK_TOKEN` environment variable, to run the tests with any org and user needed.

test/smoke/iac/test.spec.ts

@@ -0,0 +1,28 @@
+import { run } from '../../jest/acceptance/iac/helpers';
+
+jest.setTimeout(1_000 * 90);
+
+describe('snyk iac test --experimental', () => {
+  beforeAll(async () => {
+    await login();
+  });
+
+  it('runs successfully and resolves with a non-error exit code', async () => {
+    // Arrange
+    const filePath = 'iac/depth_detection/root.tf';
+
+    // Act
+    const { stderr, stdout, exitCode } = await run(
+      `snyk iac test --experimental ${filePath}`,
+    );
+
+    // Assert
+    expect(stdout).toContain('Infrastructure as Code');
+    expect(stderr).toBe('');
+    expect(exitCode).toBeLessThan(2);
+  });
+
+  async function login() {
+    await run(`snyk auth ${process.env.IAC_SMOKE_TESTS_SNYK_TOKEN}`);
+  }
+});