test: breakout additional integration tests to run in parallel

nick-fields · Aug 4, 2022 · c71afb9 · c71afb9
1 parent 5161d68
commit c71afb9
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 77 deletions.
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml
@@ -50,8 +50,6 @@ jobs:
     name: Run Integration Large Output Tests
     if: startsWith(github.ref, 'refs/heads')
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: true
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -77,9 +75,106 @@ jobs:
       - name: Assert exit code is expected
         uses: nick-invision/assert-action@v1
         with:
-          expected: 33
+          expected: 2
           actual: ${{ steps.large-output.outputs.exit_code }}
 
+  ci_integration_retry_on_exit_code:
+    name: Run Integration retry_on_exit_code Tests
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+      - name: retry_on_exit_code (with expected error code)
+        id: retry_on_exit_code_expected
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          retry_on_exit_code: 2
+          max_attempts: 3
+          command: node -e "process.exit(2)"
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.retry_on_exit_code_expected.outcome }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 3
+          actual: ${{ steps.retry_on_exit_code_expected.outputs.total_attempts }}
+
+      - name: retry_on_exit_code (with unexpected error code)
+        id: retry_on_exit_code_unexpected
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          retry_on_exit_code: 2
+          max_attempts: 3
+          command: node -e "process.exit(1)"
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.retry_on_exit_code_unexpected.outcome }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 1
+          actual: ${{ steps.retry_on_exit_code_unexpected.outputs.total_attempts }}
+
+  ci_integration_continue_on_error:
+    name: Run Integration continue_on_error Tests
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+      - name: happy-path (continue_on_error)
+        id: happy_path_continue_on_error
+        uses: ./
+        with:
+          command: node -e "process.exit(0)"
+          timeout_minutes: 1
+          continue_on_error: true
+      - name: sad-path (continue_on_error)
+        id: sad_path_continue_on_error
+        uses: ./
+        with:
+          command: node -e "process.exit(33)"
+          timeout_minutes: 1
+          continue_on_error: true
+      - name: Verify continue_on_error returns correct exit code on success
+        uses: nick-invision/assert-action@v1
+        with:
+          expected: 0
+          actual: ${{ steps.happy_path_continue_on_error.outputs.exit_code }}
+      - name: Verify continue_on_error exits with correct outcome on success
+        uses: nick-invision/assert-action@v1
+        with:
+          expected: success
+          actual: ${{ steps.happy_path_continue_on_error.outcome }}
+      - name: Verify continue_on_error returns correct exit code on error
+        uses: nick-invision/assert-action@v1
+        with:
+          expected: 33
+          actual: ${{ steps.sad_path_continue_on_error.outputs.exit_code }}
+      - name: Verify continue_on_error exits with successful outcome when an error occurs
+        uses: nick-invision/assert-action@v1
+        with:
+          expected: success
+          actual: ${{ steps.sad_path_continue_on_error.outcome }}
+
   ci_integration:
     name: Run Integration Tests
     if: startsWith(github.ref, 'refs/heads')
@@ -154,42 +249,6 @@ jobs:
           command: node -e "process.exit(1)"
           on_retry_command: node -e "console.log('this is a retry command')"
 
-      - name: retry_on_exit_code (with expected error code)
-        id: retry_on_exit_code_expected
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          retry_on_exit_code: 2
-          max_attempts: 3
-          command: node -e "process.exit(2)"
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.retry_on_exit_code_expected.outcome }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 3
-          actual: ${{ steps.retry_on_exit_code_expected.outputs.total_attempts }}
-
-      - name: retry_on_exit_code (with unexpected error code)
-        id: retry_on_exit_code_unexpected
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          retry_on_exit_code: 2
-          max_attempts: 3
-          command: node -e "process.exit(1)"
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.retry_on_exit_code_unexpected.outcome }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 1
-          actual: ${{ steps.retry_on_exit_code_unexpected.outputs.total_attempts }}
-
       - name: on-retry-cmd (on-retry fails)
         id: on-retry-cmd-fails
         uses: ./
@@ -217,41 +276,6 @@ jobs:
           expected: failure
           actual: ${{ steps.sad_path_error.outcome }}
 
-      - name: happy-path (continue_on_error)
-        id: happy_path_continue_on_error
-        uses: ./
-        with:
-          command: node -e "process.exit(0)"
-          timeout_minutes: 1
-          continue_on_error: true
-      - name: sad-path (continue_on_error)
-        id: sad_path_continue_on_error
-        uses: ./
-        with:
-          command: node -e "process.exit(33)"
-          timeout_minutes: 1
-          continue_on_error: true
-      - name: Verify continue_on_error returns correct exit code on success
-        uses: nick-invision/assert-action@v1
-        with:
-          expected: 0
-          actual: ${{ steps.happy_path_continue_on_error.outputs.exit_code }}
-      - name: Verify continue_on_error exits with correct outcome on success
-        uses: nick-invision/assert-action@v1
-        with:
-          expected: success
-          actual: ${{ steps.happy_path_continue_on_error.outcome }}
-      - name: Verify continue_on_error returns correct exit code on error
-        uses: nick-invision/assert-action@v1
-        with:
-          expected: 33
-          actual: ${{ steps.sad_path_continue_on_error.outputs.exit_code }}
-      - name: Verify continue_on_error exits with successful outcome when an error occurs
-        uses: nick-invision/assert-action@v1
-        with:
-          expected: success
-          actual: ${{ steps.sad_path_continue_on_error.outcome }}
-
       - name: retry_on (timeout) fails early if error encountered
         id: retry_on_timeout_fail
         uses: ./

diff --git a/test-data/large-output/Makefile b/test-data/large-output/Makefile
@@ -5,9 +5,9 @@ SHELL = bash
 # https://github.com/nick-fields/retry/issues/84
 
 bytes-%:
-	for i in {1..$*}; do cat kibibyte.txt; done; exit 33
+	for i in {1..$*}; do cat kibibyte.txt; done; exit 2
 .PHONY: bytes-%
 
 lines-%:
-	for i in {1..$*}; do echo a; done; exit 33
-.PHONY: lines-%
+	for i in {1..$*}; do echo a; done; exit 2
+.PHONY: lines-%