Merge pull request #618 from martin-schulze-vireso/feature/local_retry

Add BATS_TEST_RETRIES

docs/CHANGELOG.md

@@ -15,6 +15,8 @@ The format is based on [Keep a Changelog][kac] and this project adheres to
 * using external formatters via `--formatter <absolute path>` (also works for
   `--report-formatter`) (#602)
 * running only tests that failed in the last run via `--filter-status failed` (#483)
+* variable `BATS_TEST_RETRIES` that specifies how often a test should be
+  reattempted before it is considered failed (#618)
 
 #### Documentation
 

docs/source/writing-tests.md

@@ -376,6 +376,9 @@ There are several global variables you can use to introspect on Bats tests:
 - `BATS_TEST_NAME_PREFIX` will be prepended to the description of each test on 
    stdout and in reports.
 - `$BATS_TEST_DESCRIPTION` is the description of the current test case.
+- `BATS_TEST_RETRIES` is the maximum number of additional attempts that will be
+  made on a failed test before it is finally considered failed.
+  The default of 0 means the test must pass on the first attempt.
 - `$BATS_TEST_NUMBER` is the (1-based) index of the current test case in the test file.
 - `$BATS_SUITE_TEST_NUMBER` is the (1-based) index of the current test case in the test suite (over all files).
 - `$BATS_TMPDIR` is the base temporary directory used by bats to create its

lib/bats-core/test_functions.bash

@@ -348,3 +348,10 @@ bats_test_function() {
   local test_name="$1"
   BATS_TEST_NAMES+=("$test_name")
 }
+
+# decides whether a failed test should be run again
+bats_should_retry_test() {
+  # test try number starts at 1
+  # 0 retries means run only first try
+  (( BATS_TEST_TRY_NUMBER <= BATS_TEST_RETRIES ))
+}

libexec/bats-core/bats-exec-file

@@ -5,6 +5,8 @@ flags=('--dummy-flag')
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 extended_syntax=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
+declare -r BATS_RETRY_RETURN_CODE=126
+export BATS_TEST_RETRIES=0 # no retries by default 
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
@@ -118,17 +120,20 @@ source "$BATS_ROOT/lib/bats-core/common.bash"
 
 bats_file_exit_trap() {
   trap - ERR EXIT
+  local failure_reason
+  local -i failure_test_index=$(( BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + 1 ))
   if [[ -z "$BATS_SETUP_FILE_COMPLETED" || -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then
     if [[ -z "$BATS_SETUP_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='setup_file'
+      failure_reason='setup_file'
     elif [[ -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='teardown_file'
+      failure_reason='teardown_file'
+      failure_test_index=$(( BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + ${#tests_to_run[@]} + 1 ))
     elif [[ -z "$BATS_SOURCE_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='source'
+      failure_reason='source'
     else
-      FAILURE_REASON='unknown internal'
+      failure_reason='unknown internal'
     fi
-    printf "not ok %d %s\n" "$((test_number_in_suite + 1))" "$FAILURE_REASON failed" >&3
+    printf "not ok %d %s\n" "$failure_test_index" "$failure_reason failed" >&3
     local stack_trace
     bats_get_failure_stack_trace stack_trace
     bats_print_stack_trace "${stack_trace[@]}" >&3
@@ -175,11 +180,11 @@ bats_is_next_parallel_test_finished() {
 bats_forward_output_for_parallel_tests() {
   local status=0
   # was the next test already started?
-  while [[ $(( test_number_in_suite_of_last_finished_test + 1 )) -le $test_number_in_suite ]]; do
+  while (( test_number_in_suite_of_last_finished_test + 1 <= test_number_in_suite )); do
     # if we are okay with waiting or if the test has already been finished
     if [[ "$1" == "blocking" ]] || bats_is_next_parallel_test_finished ; then
       (( ++test_number_in_suite_of_last_finished_test ))
-      bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=1
+      bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=$?
     else
       # non-blocking and the process has not finished -> abort the printing
       break
@@ -188,6 +193,25 @@ bats_forward_output_for_parallel_tests() {
   return $status
 }
 
+bats_run_test_with_retries() { # <args>
+  local status=0
+  local should_try_again=1 try_number
+  for ((try_number=1; should_try_again; ++try_number)); do
+    if "$BATS_LIBEXEC/bats-exec-test" "$@" "$try_number"; then
+      should_try_again=0
+    else
+      status=$?
+      if ((status == BATS_RETRY_RETURN_CODE)); then
+        should_try_again=1
+      else
+        should_try_again=0
+        bats_exec_file_status=$status
+      fi
+    fi
+  done
+  return $status
+}
+
 bats_run_tests_in_parallel() {
   local output_folder="$BATS_RUN_TMPDIR/parallel_output"
   local status=0
@@ -196,15 +220,16 @@ bats_run_tests_in_parallel() {
   source "$BATS_ROOT/lib/bats-core/semaphore.bash"
   bats_semaphore_setup
   # the test_number_in_file is not yet incremented -> one before the next test to run
-  local test_number_in_suite_of_last_finished_test="$test_number_in_suite" # stores which test was printed last
+  local test_number_in_suite_of_last_finished_test="$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE" # stores which test was printed last
+  local test_number_in_file=0 test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE
   for test_name in "${tests_to_run[@]}"; do
     # Only handle non-empty lines
     if [[ $test_name ]]; then
       ((++test_number_in_suite))
       ((++test_number_in_file))
       mkdir -p "$output_folder/$test_number_in_suite"
       bats_semaphore_run "$output_folder/$test_number_in_suite" \
-                          "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \
+                          bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \
                           > "$output_folder/$test_number_in_suite/pid"
     fi
     # print results early to get interactive feedback
@@ -218,7 +243,7 @@ bats_read_tests_list_file() {
   local line_number=0
   tests_to_run=()
   # the global test number must be visible to traps -> not local
-  first_test_number_in_suite=''
+  local test_number_in_suite=''
   while read -r test_line; do
     # check if the line begins with filename
     # filename might contain some hard to parse characters,
@@ -229,15 +254,14 @@ bats_read_tests_list_file() {
       tests_to_run+=("$test_name")
       # save the first test's number for later iteration
       # this assumes that tests for a file are stored consecutive in the file!
-      if [[ -z "$first_test_number_in_suite" ]]; then
-        first_test_number_in_suite=$line_number
+      if [[ -z "$test_number_in_suite" ]]; then
+        test_number_in_suite=$line_number
       fi
     fi
     ((++line_number))
   done <"$TESTS_FILE"
-
-  test_number_in_suite="$first_test_number_in_suite"
-  test_number_in_file=0
+  BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE="$test_number_in_suite"
+  declare -ri BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE # mark readonly (cannot merge assignment, because value would be lost)
 }
 
 bats_run_tests() {
@@ -247,6 +271,8 @@ bats_run_tests() {
     export BATS_SEMAPHORE_NUMBER_OF_SLOTS="$num_jobs"
     bats_run_tests_in_parallel "$BATS_RUN_TMPDIR/parallel_output" || bats_exec_file_status=1
   else
+    local test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE \
+          test_number_in_file=0
     for test_name in "${tests_to_run[@]}"; do
       if [[ "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
         bats_exec_file_status=130 # bash's code for SIGINT exits
@@ -256,7 +282,8 @@ bats_run_tests() {
       if [[ $test_name ]]; then
         ((++test_number_in_suite))
         ((++test_number_in_file))
-        "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1
+        bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" \
+                                    "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=$?
       fi
     done
   fi
@@ -268,7 +295,7 @@ bats_create_file_tempdirs() {
     printf 'Failed to create %s\n' "$bats_files_tmpdir" >&2
     exit 1
   fi
-  BATS_FILE_TMPDIR="$bats_files_tmpdir/$first_test_number_in_suite"
+  BATS_FILE_TMPDIR="$bats_files_tmpdir/${BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE?}"
   if ! mkdir "$BATS_FILE_TMPDIR"; then
     printf 'Failed to create BATS_FILE_TMPDIR=%s\n' "$BATS_FILE_TMPDIR" >&2
     exit 1
@@ -283,6 +310,7 @@ if [[ -n "$extended_syntax" ]]; then
   printf "suite %s\n" "$filename"
 fi
 
+BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE=0 # predeclare as Bash 3.2 does not support declare -g
 bats_read_tests_list_file
 
 # don't run potentially expensive setup/teardown_file

libexec/bats-core/bats-exec-test

@@ -49,6 +49,7 @@ export BATS_TEST_FILENAME="$1"
 export BATS_TEST_NAME="$2"
 export BATS_SUITE_TEST_NUMBER="$3"
 export BATS_TEST_NUMBER="$4"
+BATS_TEST_TRY_NUMBER="$5"
 
 if [[ -z "$BATS_TEST_FILENAME" ]]; then
   printf 'usage: bats-exec-test <filename>\n' >&2
@@ -67,7 +68,7 @@ bats_create_test_tmpdirs() {
 
   BATS_TEST_TMPDIR="$tests_tmpdir/$BATS_SUITE_TEST_NUMBER"
   if ! mkdir "$BATS_TEST_TMPDIR"; then
-    printf 'Failed to create BATS_TEST_TMPDIR: %s\n' "$BATS_TEST_TMPDIR" >&2
+    printf 'Failed to create BATS_TEST_TMPDIR%d: %s\n' "$BATS_TEST_TRY_NUMBER" "$BATS_TEST_TMPDIR" >&2
     exit 1
   fi
 
@@ -123,6 +124,7 @@ bats_exit_trap() {
 
   local print_bats_out="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS}"
 
+  local should_retry=''
   if [[ -z "$BATS_TEST_COMPLETED" || -z "$BATS_TEARDOWN_COMPLETED" || "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
     if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then
       # For some versions of bash, `$?` may not be set properly for some error
@@ -136,36 +138,46 @@ bats_exit_trap() {
       # output, since there's no way to reach the `bats_exit_trap` call.
       BATS_ERROR_STATUS=1
     fi
-    printf 'not ok %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3
-    local stack_trace
-    bats_get_failure_stack_trace stack_trace
-    bats_print_stack_trace "${stack_trace[@]}" >&3
-    bats_print_failed_command "${stack_trace[@]}" >&3
-
-    if [[ $BATS_PRINT_OUTPUT_ON_FAILURE && -n "${output:-}" ]]; then
-      printf "Last output:\n%s\n" "$output" >> "$BATS_OUT"
-    fi 
-
-    print_bats_out=1
-    status=1
-    local state=failed
+    if bats_should_retry_test; then
+      should_retry=1
+      status=126 # signify retry
+      rm -r "$BATS_TEST_TMPDIR" # clean up for retry
+    else
+      printf 'not ok %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3
+      local stack_trace
+      bats_get_failure_stack_trace stack_trace
+      bats_print_stack_trace "${stack_trace[@]}" >&3
+      bats_print_failed_command "${stack_trace[@]}" >&3
+
+      if [[ $BATS_PRINT_OUTPUT_ON_FAILURE && -n "${output:-}" ]]; then
+        printf "Last output:\n%s\n" "$output" >> "$BATS_OUT"
+      fi 
+
+      print_bats_out=1    
+      status=1
+      local state=failed
+    fi
   else
     printf 'ok %d %s%s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" \
       "$skipped" >&3
     status=0
     local state=passed
   fi
 
-  printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "$BATS_RUNLOG_FILE"
+  if [[ -z "$should_retry" ]]; then
+    printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "$BATS_RUNLOG_FILE"
 
-  if [[ $print_bats_out ]]; then
-    bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3
+    if [[ $print_bats_out ]]; then
+      bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3
+    fi
   fi
-
   if [[ $BATS_GATHER_TEST_OUTPUTS_IN ]]; then
-    cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER-$BATS_TEST_DESCRIPTION.log"
+    local try_suffix=
+    if [[ -n "$should_retry" ]]; then
+      try_suffix="-try$BATS_TEST_TRY_NUMBER"
+    fi
+    cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER$try_suffix-$BATS_TEST_DESCRIPTION.log"
   fi
-
   rm -f "$BATS_OUT"
   exit "$status"
 }

man/bats.7

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "BATS" "7" "May 2022" "bats-core" "Bash Automated Testing System"
+.TH "BATS" "7" "July 2022" "bats-core" "Bash Automated Testing System"
 .
 .SH "NAME"
 \fBbats\fR \- Bats test file format
@@ -33,7 +33,7 @@ A Bats test file is a Bash script with special syntax for defining test cases\.
 Each Bats test file is evaluated n+1 times, where \fIn\fR is the number of test cases in the file\. The first run counts the number of test cases, then iterates over the test cases and executes each one in its own process\.
 .
 .SH "THE RUN HELPER"
-Usage: run [OPTIONS] [\-\-] <command\.\.\.> Options: ! check for non zero exit code \-\fIN\fR check that exit code is \fIN\fR \-\-separate\-stderr split stderr and stdout \-\-keep\-empty\-lines retain emtpy lines in \fB${lines[@]}\fR/\fB${stderr_lines[@]}\fR
+Usage: run [OPTIONS] [\-\-] <command\.\.\.> Options: ! check for non zero exit code \-\fIN\fR check that exit code is \fIN\fR \-\-separate\-stderr split stderr and stdout \-\-keep\-empty\-lines retain empty lines in \fB${lines[@]}\fR/\fB${stderr_lines[@]}\fR
 .
 .P
 Many Bats tests need to run a command and then make assertions about its exit status and output\. Bats includes a \fBrun\fR helper that invokes its arguments as a command, saves the exit status and output into special global variables, and (optionally) checks exit status against a given expected value\. If successful, \fBrun\fR returns with a \fB0\fR status code so you can continue to make assertions in your test case\.
@@ -266,6 +266,9 @@ There are several global variables you can use to introspect on Bats tests:
 \fB$BATS_TEST_DESCRIPTION\fR is the description of the current test case\.
 .
 .IP "\(bu" 4
+\fBBATS_TEST_RETRIES\fR is the maximum number of additional attempts that will be made on a failed test before it is finally considered failed\. The default of 0 means the test must pass on the first attempt\.
+.
+.IP "\(bu" 4
 \fB$BATS_TEST_NUMBER\fR is the (1\-based) index of the current test case in the test file\.
 .
 .IP "\(bu" 4

man/bats.7.ronn

@@ -235,6 +235,9 @@ test case.
 on stdout and in reports.
 * `$BATS_TEST_DESCRIPTION` is the description of the current test
 case.
+* `BATS_TEST_RETRIES` is the maximum number of additional attempts that will be
+  made on a failed test before it is finally considered failed.
+  The default of 0 means the test must pass on the first attempt.
 * `$BATS_TEST_NUMBER` is the (1-based) index of the current test case
 in the test file.
 * `$BATS_SUITE_TEST_NUMBER` is the (1-based) index of the current test

test/bats.bats

@@ -802,6 +802,8 @@ END_OF_ERR_MSG
 }
 
 @test "CTRL-C aborts and fails after run" {
+  # shellcheck disable=SC2034
+  BATS_TEST_RETRIES=2
   if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then
     skip "Aborts don't work in parallel mode"
   fi
@@ -1364,3 +1366,52 @@ enforce_own_process_group() {
   run find .bats/run-logs -name '*.log'
   [ "$first_run_logs" == "$output" ]
 }
+
+@test "BATS_TEST_RETRIES allows for retrying tests" {
+  export LOG="$BATS_TEST_TMPDIR/call.log"
+  bats_require_minimum_version 1.5.0
+  run ! bats "$FIXTURE_ROOT/retry.bats"
+  [ "${lines[0]}" == '1..3' ]
+  [ "${lines[1]}" == 'not ok 1 Fail all' ]
+  [ "${lines[4]}" == 'ok 2 Fail once' ]
+  [ "${lines[5]}" == 'not ok 3 Override retries' ]
+
+  run cat "$LOG"
+  [ "${lines[0]}" == ' setup_file ' ] # should only be executed once
+  [ "${lines[22]}" == ' teardown_file ' ] # should only be executed once
+  [ "${#lines[@]}" -eq 23 ]
+
+  # 3x Fail All (give up after 3 tries/2 retries)
+  run grep test_Fail_all < "$LOG"
+  [ "${lines[0]}" == 'test_Fail_all setup 1' ] # should be executed for each try
+  [ "${lines[1]}" == 'test_Fail_all test_Fail_all 1' ]
+  [ "${lines[2]}" == 'test_Fail_all teardown 1' ] # should be executed for each try
+  [ "${lines[3]}" == 'test_Fail_all setup 2' ]
+  [ "${lines[4]}" == 'test_Fail_all test_Fail_all 2' ]
+  [ "${lines[5]}" == 'test_Fail_all teardown 2' ]
+  [ "${lines[6]}" == 'test_Fail_all setup 3' ]
+  [ "${lines[7]}" == 'test_Fail_all test_Fail_all 3' ]
+  [ "${lines[8]}" == 'test_Fail_all teardown 3' ]
+  [ "${#lines[@]}" -eq 9 ]
+
+  # 2x Fail once (pass second try/first retry)
+  run grep test_Fail_once < "$LOG"
+  [ "${lines[0]}" == 'test_Fail_once setup 1' ]
+  [ "${lines[1]}" == 'test_Fail_once test_Fail_once 1' ]
+  [ "${lines[2]}" == 'test_Fail_once teardown 1' ]
+  [ "${lines[3]}" == 'test_Fail_once setup 2' ]
+  [ "${lines[4]}" == 'test_Fail_once test_Fail_once 2' ]
+  [ "${lines[5]}" == 'test_Fail_once teardown 2' ]
+  [ "${#lines[@]}" -eq 6 ]
+
+  # 2x Override retries (give up after second try/first retry)
+  run grep test_Override_retries < "$LOG"
+  [ "${lines[0]}" == 'test_Override_retries setup 1' ]
+  [ "${lines[1]}" == 'test_Override_retries test_Override_retries 1' ]
+  [ "${lines[2]}" == 'test_Override_retries teardown 1' ]
+  [ "${lines[3]}" == 'test_Override_retries setup 2' ]
+  [ "${lines[4]}" == 'test_Override_retries test_Override_retries 2' ]
+  [ "${lines[5]}" == 'test_Override_retries teardown 2' ]
+  [ "${#lines[@]}" -eq 6 ]
+
+}