Skip to content

Commit

Permalink
Refactor to make testing easier (#90)
Browse files Browse the repository at this point in the history 
* minor: refactor to make testing easier

* patch: retrieve inputs into object rather than globals

* test: run more "integration" tests in parallel

* test: fix needs and rearrange ci_integration_* jobs

* test: forgot comma

* test: fix sad_path_timeout_minutes assertions

* test: add single ci_all_tests_passed job that can be required for CI rather than each individual job

* test: add single ci_all_tests_passed job that can be required for CI rather than each individual job
  • Loading branch information
@nick-fields
nick-fields committed Aug 6, 2022
1 parent 616fa81 commit b4fa575
Show file tree
Hide file tree
Showing 5 changed files with 525 additions and 312 deletions.
300 changes: 196 additions & 104 deletions .github/workflows/ci_cd.yml
View file
Expand Up @@ -24,6 +24,117 @@ jobs:
directory: ./coverage/
verbose: true

ci_integration:
name: Run Integration Tests
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: happy-path
id: happy_path
uses: ./
with:
timeout_minutes: 1
max_attempts: 2
command: npm -v
- uses: nick-invision/assert-action@v1
with:
expected: true
actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}

- name: log examples
uses: ./
with:
command: node ./.github/scripts/log-examples.js
timeout_minutes: 1

- name: sad-path (error)
id: sad_path_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
command: node -e "process.exit(1)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_error.outcome }}

- name: retry_on (timeout) fails early if error encountered
id: retry_on_timeout_fail
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 3
retry_on: timeout
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 1
actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_timeout_fail.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}

- name: retry_on (error)
id: retry_on_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
retry_on: error
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_error.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.exit_code }}

- name: sad-path (wrong shell for OS)
id: wrong_shell
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
shell: cmd
command: 'dir'
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.wrong_shell.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.wrong_shell.outcome }}

ci_integration_envvar:
name: Run Integration Env Var Tests
if: startsWith(github.ref, 'refs/heads')
Expand Down Expand Up @@ -175,8 +286,8 @@ jobs:
expected: success
actual: ${{ steps.sad_path_continue_on_error.outcome }}

ci_integration:
name: Run Integration Tests
ci_integration_retry_wait_seconds:
name: Run Integration Tests (retry_wait_seconds)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
Expand All @@ -189,24 +300,6 @@ jobs:
- name: Install dependencies
run: npm ci

- name: happy-path
id: happy_path
uses: ./
with:
timeout_minutes: 1
max_attempts: 2
command: npm -v
- uses: nick-invision/assert-action@v1
with:
expected: true
actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}

- name: log examples
uses: ./
with:
command: node ./.github/scripts/log-examples.js
timeout_minutes: 1

- name: sad-path (retry_wait_seconds)
id: sad_path_wait_sec
uses: ./
Expand All @@ -230,6 +323,20 @@ jobs:
actual: ${{ steps.sad_path_wait_sec.outputs.exit_error }}
comparison: contains

ci_integration_on_retry_cmd:
name: Run Integration Tests (on_retry_command)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: new-command-on-retry
id: new-command-on-retry
uses: ./
Expand Down Expand Up @@ -259,88 +366,9 @@ jobs:
command: node -e "process.exit(1)"
on_retry_command: node -e "throw new Error('This is an on-retry command error')"

- name: sad-path (error)
id: sad_path_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
command: node -e "process.exit(1)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_error.outcome }}

- name: retry_on (timeout) fails early if error encountered
id: retry_on_timeout_fail
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 3
retry_on: timeout
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 1
actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_timeout_fail.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}

- name: retry_on (error)
id: retry_on_error
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
retry_on: error
command: node -e "process.exit(2)"
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.retry_on_error.outcome }}
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.retry_on_error.outputs.exit_code }}

- name: sad-path (wrong shell for OS)
id: wrong_shell
uses: ./
continue-on-error: true
with:
timeout_minutes: 1
max_attempts: 2
shell: cmd
command: 'dir'
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.wrong_shell.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.wrong_shell.outcome }}

# timeout tests take longer to run so run in parallel
ci_integration_timeout:
name: Run Integration Timeout Tests
ci_integration_timeout_seconds:
name: Run Integration Timeout Tests (seconds)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -370,6 +398,20 @@ jobs:
expected: failure
actual: ${{ steps.sad_path_timeout.outcome }}

ci_integration_timeout_retry_on_timeout:
name: Run Integration Timeout Tests (retry_on timeout)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: retry_on (timeout)
id: retry_on_timeout
uses: ./
Expand All @@ -388,6 +430,20 @@ jobs:
expected: failure
actual: ${{ steps.retry_on_timeout.outcome }}

ci_integration_timeout_retry_on_error:
name: Run Integration Timeout Tests (retry_on error)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: retry_on (error) fails early if timeout encountered
id: retry_on_error_fail
uses: ./
Expand All @@ -410,6 +466,20 @@ jobs:
expected: 1
actual: ${{ steps.retry_on_error_fail.outputs.exit_code }}

ci_integration_timeout_minutes:
name: Run Integration Timeout Tests (minutes)
if: startsWith(github.ref, 'refs/heads')
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- name: Install dependencies
run: npm ci

- name: sad-path (timeout minutes)
id: sad_path_timeout_minutes
uses: ./
Expand All @@ -421,11 +491,11 @@ jobs:
- uses: nick-invision/assert-action@v1
with:
expected: 2
actual: ${{ steps.sad_path_timeout.outputs.total_attempts }}
actual: ${{ steps.sad_path_timeout_minutes.outputs.total_attempts }}
- uses: nick-invision/assert-action@v1
with:
expected: failure
actual: ${{ steps.sad_path_timeout.outcome }}
actual: ${{ steps.sad_path_timeout_minutes.outcome }}

ci_windows:
name: Run Windows Tests
Expand Down Expand Up @@ -479,10 +549,32 @@ jobs:
echo "this is
a test"
ci_all_tests_passed:
name: All tests passed
needs:
[
ci_unit,
ci_integration,
ci_integration_envvar,
ci_integration_large_output,
ci_integration_on_retry_cmd,
ci_integration_retry_wait_seconds,
ci_integration_continue_on_error,
ci_integration_retry_on_exit_code,
ci_integration_timeout_seconds,
ci_integration_timeout_minutes,
ci_integration_timeout_retry_on_timeout,
ci_integration_timeout_retry_on_error,
ci_windows,
]
runs-on: ubuntu-latest
steps:
- run: echo "If this is hit, all tests successfully passed"

# runs on push to master only
cd:
name: Publish Action
needs: [ci_integration, ci_integration_timeout, ci_windows]
needs: [ci_all_tests_passed]
if: github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
steps:
Expand Down

0 comments on commit b4fa575

Please sign in to comment.