Skip to content

Commit

Permalink
chore: Create IaC smoke tests for experimental test
Browse files Browse the repository at this point in the history
  • Loading branch information
ofekatr committed Sep 20, 2022
1 parent a3f87ba commit 8890fb3
Show file tree
Hide file tree
Showing 6 changed files with 231 additions and 1 deletion.
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ test/fixtures/container-app-vulns/ @snyk/mycelium
test/fixtures/container-projects/ @snyk/mycelium @snyk/potion
test/fixtures/docker/ @snyk/mycelium @snyk/potion
test/fixtures/iac/ @snyk/group-infrastructure-as-code
test/smoke/iac/ @snyk/group-infrastructure-as-code
test/smoke/spec/iac/ @snyk/group-infrastructure-as-code
test/smoke/spec/snyk_code_spec.sh @snyk/zenith
test/smoke/spec/snyk_basic_spec.sh @snyk/hammer
Expand Down Expand Up @@ -70,6 +71,8 @@ test/fixtures/unmanaged-log4j-fixture @snyk/tundra
test/jest/acceptance/snyk-log4shell/log4shell-detection.spec.ts @snyk/tundra
test/jest/acceptance/snyk-test/app-vuln-container-project.spec.ts @snyk/mycelium
/.github @snyk/hammer
/.github/workflows/iac-smoke-tests.yml @snyk/group-infrastructure-as-code
/.github/workflows/iac-smoke-tests-pulls.yml @snyk/group-infrastructure-as-code

# tap tests ownership
test/tap/cli-monitor/ @snyk/snyk-open-source
Expand Down
92 changes: 92 additions & 0 deletions .github/workflows/iac-smoke-tests-pulls.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
name: Infrastructure as Code Smoke Tests (Pull Requests)

on:
pull_request:
branches: [master]

jobs:
check_for_changed_iac_files:
name: Check for changed IaC files
runs-on: ubuntu-latest
outputs:
is_changed: ${{ steps.check_iac_files_changed.outputs.is_changed }}

steps:
- uses: actions/checkout@v2

- name: Install jq
run: |
sudo apt-get install jq
- name: Parse CODEOWNERS file
id: codeowners
uses: SvanBoxel/codeowners-action@v2.2
with:
file_match_info: 'true'
path: ./.github/CODEOWNERS

- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v29.0.4

- name: Get all IaC files
id: get_all_iac_files
run: |
ALL_IAC_FILES=$(
echo ${{ toJSON(steps.codeowners.outputs.filematches) }} |
jq '[
to_entries[] |
select(
.value.owners |
index("@snyk/group-infrastructure-as-code")
) |
.key
]'
)
echo "::set-output name=all_iac_files::$(
echo $ALL_IAC_FILES
)"
- id: check_iac_files_changed
name: Check for changed files owned by IaC
run: |
ALL_IAC_FILES=${{ toJson(steps.get_all_iac_files.outputs.all_iac_files) }}
CHANGED_FILES=$(
echo ${{ steps.changed-files.outputs.all_changed_files }} |
jq -R 'split(" ")'
)
CHANGED_IAC_FILES=$(
echo $CHANGED_FILES |
jq --argjson ALL_IAC_FILES "$ALL_IAC_FILES" '[
.[] |
. as $changed_file |
select(
$ALL_IAC_FILES |
index($changed_file)
)
]'
)
CHANGED_IAC_FILES_COUNT=$(
echo $CHANGED_IAC_FILES | jq 'length'
)
IS_CHANGED=$(
echo $CHANGED_IAC_FILES_COUNT | jq '. > 0'
)
$IS_CHANGED &&
echo "Found $CHANGED_IAC_FILES_COUNT changed IaC files: $CHANGED_IAC_FILES"||
echo "No changed IaC files found!"
echo "::set-output name=is_changed::$IS_CHANGED"
run_iac_smoke_tests:
name: Run IaC smoke tests
uses: ./.github/workflows/iac-smoke-tests.yml
needs: check_for_changed_iac_files
if: ${{ needs.check_for_changed_iac_files.outputs.is_changed == 'true' }}
secrets: inherit
67 changes: 67 additions & 0 deletions .github/workflows/iac-smoke-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: Infrastructure as Code Smoke Tests

on:
schedule:
- cron: '0 * * * *'
release:
types: [published]
workflow_call:

jobs:
run_iac_e2e_tests:
runs-on: ${{ matrix.os }}-latest
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]

steps:
- uses: actions/checkout@v2
with:
ref: ${{ github.ref }}

- uses: actions/setup-node@v3
with:
node-version: 15

- name: Install jq on macOS
if: ${{ matrix.os == 'macos' }}
run: |
brew install jq
- name: Install jq on Windows
if: ${{ matrix.os == 'windows'}}
run: |
iwr -useb get.scoop.sh -outfile 'install-scoop.ps1'
.\install-scoop.ps1 -RunAsAdmin
scoop install jq
- name: Install jq on Ubuntu
if: ${{ matrix.os == 'ubuntu' }}
run: |
sudo apt-get install jq
- name: Install dependencies
run: |
npm install
- name: Build Snyk CLI
run: |
npm run build
- name: Run IaC smoke tests - non-Windows
if: ${{ matrix.os != 'windows' }}
env:
IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
TEST_SNYK_COMMAND: ${{ format('node {0}/dist/cli/index.js', github.workspace) }}
run: |
npm run test:smoke:iac
- name: Run IaC smoke tests - Windows
if: ${{ matrix.os == 'windows' }}
shell: pwsh
env:
IAC_SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.IAC_SMOKE_TESTS_SNYK_TOKEN }}
TEST_SNYK_COMMAND: ${{ format('node {0}\dist\cli\index.js', github.workspace) }}
run: |
npm run test:smoke:iac
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@
"test:unit": "jest --runInBand --testPathPattern '/test(/jest)?/unit/'",
"test:acceptance": "jest --runInBand --testPathPattern '/test(/jest)?/acceptance/'",
"test:tap": "tap -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register test/tap/*.test.* ",
"test:smoke": "./scripts/run-smoke-tests-locally.sh"
"test:smoke": "./scripts/run-smoke-tests-locally.sh",
"test:smoke:iac": "jest --runInBand --testPathPattern '/test/smoke(/jest)?/iac/'"
},
"keywords": [
"security",
Expand Down
39 changes: 39 additions & 0 deletions test/smoke/iac/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Snyk Infrastructure as Code Smoke Tests

Design goal is to have a single test suite, aligned with the scope of the Snyk CLI's smoke tests, that can detect if IaC commands do not work properly - before and after it's released. The tests help us incorporate resources and requests made via network calls, to provide better coverage for the end to end flow of these commands. Some examples:

- Network calls made for fetching org properties, such as feature flags data, custom severities, etc.
- Downloading resources from CDNs, e.g., binary executables, ruleset bundles, etc.

The tests were written with Jest, and use a Snyk CLI executable either configured in the PATH environment variable, or overrode, using the `TEST_SNYK_COMMAND` environment variable, see more in the 'Notes on the

# Implementation details and usage

These smoke tests are written with Jest, using the Snyk CLI executable identified on the runtime environment (See 'Notes on the local run' section below to read on how to override it)

Spec in this folder is used as a

1. **"name: Infrastructure as Code Smoke Tests" Github Action** - these run every hour and upon releases.
2. **["Infrastructure as Code Smoke Tests (Pull Requests)"] GitHub Action** - these run for pull requests to the `master` branch which include changes to files owned by group IaC.

```sh
npm run test:smoke:iac
```

### Notes on the local run

These tests can be executed with the following npm script:

```
npm run test:smoke:iac
```

Alternatively, they can be executed directly via `jest`, by running:

```
npx jest test/smoke/iac/
```

You may specify any executable that will be used by the smoke tests, by configuring the `TEST_SNYK_COMMAND` environment variable. E.g. a local exuctable `TEST_SNYK_COMMAND="./snyk-macos"` or an `TEST_SNYK_COMMAND="npx snyk@1.500.0"` or `TEST_SNYK_COMMAND="node ./dist/cli"` for local execution.

You may also configure an authentication token with the `SNYK_TOKEN` environment variable, to run the tests with any org and user needed.
28 changes: 28 additions & 0 deletions test/smoke/iac/test.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { run } from '../../jest/acceptance/iac/helpers';

jest.setTimeout(1_000 * 90);

describe('snyk iac test --experimental', () => {
beforeAll(async () => {
await login();
});

it('runs successfully and resolves with a non-error exit code', async () => {
// Arrange
const filePath = 'iac/depth_detection/root.tf';

// Act
const { stderr, stdout, exitCode } = await run(
`snyk iac test --experimental ${filePath}`,
);

// Assert
expect(stdout).toContain('Infrastructure as Code');
expect(stderr).toBe('');
expect(exitCode).toBeLessThan(2);
});

async function login() {
await run(`snyk auth ${process.env.IAC_SMOKE_TESTS_SNYK_TOKEN}`);
}
});

0 comments on commit 8890fb3

Please sign in to comment.