Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor to make testing easier #90

Merged
merged 8 commits into from Aug 6, 2022
Merged

Refactor to make testing easier #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3e77775
minor: refactor to make testing easier
nick-fields Aug 6, 2022
e4c5090
patch: retrieve inputs into object rather than globals
nick-fields Aug 6, 2022
72a0d87
test: run more "integration" tests in parallel
nick-fields Aug 6, 2022
19a1f32
test: fix needs and rearrange ci_integration_* jobs
nick-fields Aug 6, 2022
24a86d4
test: forgot comma
nick-fields Aug 6, 2022
e5ee3eb
test: fix sad_path_timeout_minutes assertions
nick-fields Aug 6, 2022
e0f728a
test: add single ci_all_tests_passed job that can be required for CI …
nick-fields Aug 6, 2022
b3fd951
test: add single ci_all_tests_passed job that can be required for CI …
nick-fields Aug 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
300 changes: 196 additions & 104 deletions .github/workflows/ci_cd.yml
View file
Open in desktop
Expand Up @@ -24,6 +24,117 @@ jobs:
directory: ./coverage/
verbose: true

ci_integration:
name: Run Integration Tests
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: happy-path
id: happy_path
uses: ./
with:
timeout_minutes: 1
max_attempts: 2
command: npm -v
- uses: nick-invision/assert-action@v1
with:
expected: true
actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}

- name: log examples
uses: ./
with:
command: node ./.github/scripts/log-examples.js
timeout_minutes: 1

- name: sad-path (error)
id: sad_path_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
command: node -e "process.exit(1)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_error.outcome }}

- name: retry_on (timeout) fails early if error encountered
id: retry_on_timeout_fail
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 3
retry_on: timeout
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 1
actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_timeout_fail.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}

- name: retry_on (error)
id: retry_on_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
retry_on: error
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_error.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.exit_code }}

- name: sad-path (wrong shell for OS)
id: wrong_shell
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
shell: cmd
command: 'dir'
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.wrong_shell.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.wrong_shell.outcome }}

ci_integration_envvar:
name: Run Integration Env Var Tests
if: startsWith(github.ref, 'refs/heads')
Expand Down Expand Up @@ -175,8 +286,8 @@ jobs:
expected: success
actual: ${{ steps.sad_path_continue_on_error.outcome }}

ci_integration:
name: Run Integration Tests
ci_integration_retry_wait_seconds:
name: Run Integration Tests (retry_wait_seconds)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
Expand All @@ -189,24 +300,6 @@ jobs:
- name: Install dependencies
run: npm ci

- name: happy-path
id: happy_path
uses: ./
with:
timeout_minutes: 1
max_attempts: 2
command: npm -v
- uses: nick-invision/assert-action@v1
with:
expected: true
actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}

- name: log examples
uses: ./
with:
command: node ./.github/scripts/log-examples.js
timeout_minutes: 1

- name: sad-path (retry_wait_seconds)
id: sad_path_wait_sec
uses: ./
Expand All @@ -230,6 +323,20 @@ jobs:
actual: ${{ steps.sad_path_wait_sec.outputs.exit_error }}
comparison: contains

ci_integration_on_retry_cmd:
name: Run Integration Tests (on_retry_command)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: new-command-on-retry
id: new-command-on-retry
uses: ./
Expand Down Expand Up @@ -259,88 +366,9 @@ jobs:
command: node -e "process.exit(1)"
on_retry_command: node -e "throw new Error('This is an on-retry command error')"

- name: sad-path (error)
id: sad_path_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
command: node -e "process.exit(1)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_error.outcome }}

- name: retry_on (timeout) fails early if error encountered
id: retry_on_timeout_fail
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 3
retry_on: timeout
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 1
actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_timeout_fail.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}

- name: retry_on (error)
id: retry_on_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
retry_on: error
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_error.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.exit_code }}

- name: sad-path (wrong shell for OS)
id: wrong_shell
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
shell: cmd
command: 'dir'
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.wrong_shell.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.wrong_shell.outcome }}

# timeout tests take longer to run so run in parallel
ci_integration_timeout:
name: Run Integration Timeout Tests
ci_integration_timeout_seconds:
name: Run Integration Timeout Tests (seconds)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -370,6 +398,20 @@ jobs:
expected: failure
actual: ${{ steps.sad_path_timeout.outcome }}

ci_integration_timeout_retry_on_timeout:
name: Run Integration Timeout Tests (retry_on timeout)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: retry_on (timeout)
id: retry_on_timeout
uses: ./
Expand All @@ -388,6 +430,20 @@ jobs:
expected: failure
actual: ${{ steps.retry_on_timeout.outcome }}

ci_integration_timeout_retry_on_error:
name: Run Integration Timeout Tests (retry_on error)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: retry_on (error) fails early if timeout encountered
id: retry_on_error_fail
uses: ./
Expand All @@ -410,6 +466,20 @@ jobs:
expected: 1
actual: ${{ steps.retry_on_error_fail.outputs.exit_code }}

ci_integration_timeout_minutes:
name: Run Integration Timeout Tests (minutes)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: sad-path (timeout minutes)
id: sad_path_timeout_minutes
uses: ./
Expand All @@ -421,11 +491,11 @@ jobs:
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_timeout.outputs.total_attempts }}
actual: ${{ steps.sad_path_timeout_minutes.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_timeout.outcome }}
actual: ${{ steps.sad_path_timeout_minutes.outcome }}

ci_windows:
name: Run Windows Tests
Expand Down Expand Up @@ -479,10 +549,32 @@ jobs:
echo "this is
a test"

ci_all_tests_passed:
name: All tests passed
needs:
[
ci_unit,
ci_integration,
ci_integration_envvar,
ci_integration_large_output,
ci_integration_on_retry_cmd,
ci_integration_retry_wait_seconds,
ci_integration_continue_on_error,
ci_integration_retry_on_exit_code,
ci_integration_timeout_seconds,
ci_integration_timeout_minutes,
ci_integration_timeout_retry_on_timeout,
ci_integration_timeout_retry_on_error,
ci_windows,
]
runs-on: ubuntu-latest
steps:
- run: echo "If this is hit, all tests successfully passed"

# runs on push to master only
cd:
name: Publish Action
needs: [ci_integration, ci_integration_timeout, ci_windows]
needs: [ci_all_tests_passed]
if: github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
steps:
Expand Down