Add BATS_TEST_RETRIES

lib/bats-core/test_functions.bash

@@ -348,3 +348,10 @@ bats_test_function() {
   local test_name="$1"
   BATS_TEST_NAMES+=("$test_name")
 }
+
+# decides whether a failed test should be run again
+bats_should_retry_test() {
+  # test try number starts at 1
+  # 0 retries means run only first try
+  (( BATS_TEST_TRY_NUMBER <= BATS_TEST_RETRIES ))
+}

libexec/bats-core/bats-exec-file

@@ -5,6 +5,8 @@ flags=('--dummy-flag')
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 extended_syntax=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
+declare -r BATS_RETRY_RETURN_CODE=126
+export BATS_TEST_RETRIES=0 # no retries by default 
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
@@ -118,17 +120,20 @@ source "$BATS_ROOT/lib/bats-core/common.bash"
 
 bats_file_exit_trap() {
   trap - ERR EXIT
+  local failure_reason
+  local -i failure_test_index=$(( BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + 1 ))
   if [[ -z "$BATS_SETUP_FILE_COMPLETED" || -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then
     if [[ -z "$BATS_SETUP_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='setup_file'
+      failure_reason='setup_file'
     elif [[ -z "$BATS_TEARDOWN_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='teardown_file'
+      failure_reason='teardown_file'
+      failure_test_index=$(( BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE + ${#tests_to_run[@]} + 1 ))
     elif [[ -z "$BATS_SOURCE_FILE_COMPLETED" ]]; then
-      FAILURE_REASON='source'
+      failure_reason='source'
     else
-      FAILURE_REASON='unknown internal'
+      failure_reason='unknown internal'
     fi
-    printf "not ok %d %s\n" "$((test_number_in_suite + 1))" "$FAILURE_REASON failed" >&3
+    printf "not ok %d %s\n" "$failure_test_index" "$failure_reason failed" >&3
     local stack_trace
     bats_get_failure_stack_trace stack_trace
     bats_print_stack_trace "${stack_trace[@]}" >&3
@@ -175,11 +180,11 @@ bats_is_next_parallel_test_finished() {
 bats_forward_output_for_parallel_tests() {
   local status=0
   # was the next test already started?
-  while [[ $(( test_number_in_suite_of_last_finished_test + 1 )) -le $test_number_in_suite ]]; do
+  while (( test_number_in_suite_of_last_finished_test + 1 <= test_number_in_suite )); do
     # if we are okay with waiting or if the test has already been finished
     if [[ "$1" == "blocking" ]] || bats_is_next_parallel_test_finished ; then
       (( ++test_number_in_suite_of_last_finished_test ))
-      bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=1
+      bats_forward_output_of_parallel_test "$test_number_in_suite_of_last_finished_test" || status=$?
     else
       # non-blocking and the process has not finished -> abort the printing
       break
@@ -188,6 +193,25 @@ bats_forward_output_for_parallel_tests() {
   return $status
 }
 
+bats_run_test_with_retries() { # <args>
+  local status=0
+  local should_try_again=1 try_number
+  for ((try_number=1; should_try_again; ++try_number)); do
+    if "$BATS_LIBEXEC/bats-exec-test" "$@" "$try_number"; then
+      should_try_again=0
+    else
+      status=$?
+      if ((status == BATS_RETRY_RETURN_CODE)); then
+        should_try_again=1
+      else
+        should_try_again=0
+        bats_exec_file_status=$status
+      fi
+    fi
+  done
+  return $status
+}
+
 bats_run_tests_in_parallel() {
   local output_folder="$BATS_RUN_TMPDIR/parallel_output"
   local status=0
@@ -196,15 +220,16 @@ bats_run_tests_in_parallel() {
   source "$BATS_ROOT/lib/bats-core/semaphore.bash"
   bats_semaphore_setup
   # the test_number_in_file is not yet incremented -> one before the next test to run
-  local test_number_in_suite_of_last_finished_test="$test_number_in_suite" # stores which test was printed last
+  local test_number_in_suite_of_last_finished_test="$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE" # stores which test was printed last
+  local test_number_in_file=0 test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE
   for test_name in "${tests_to_run[@]}"; do
     # Only handle non-empty lines
     if [[ $test_name ]]; then
       ((++test_number_in_suite))
       ((++test_number_in_file))
       mkdir -p "$output_folder/$test_number_in_suite"
       bats_semaphore_run "$output_folder/$test_number_in_suite" \
-                          "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \
+                          bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" \
                           > "$output_folder/$test_number_in_suite/pid"
     fi
     # print results early to get interactive feedback
@@ -218,7 +243,7 @@ bats_read_tests_list_file() {
   local line_number=0
   tests_to_run=()
   # the global test number must be visible to traps -> not local
-  first_test_number_in_suite=''
+  local test_number_in_suite=''
   while read -r test_line; do
     # check if the line begins with filename
     # filename might contain some hard to parse characters,
@@ -229,15 +254,14 @@ bats_read_tests_list_file() {
       tests_to_run+=("$test_name")
       # save the first test's number for later iteration
       # this assumes that tests for a file are stored consecutive in the file!
-      if [[ -z "$first_test_number_in_suite" ]]; then
-        first_test_number_in_suite=$line_number
+      if [[ -z "$test_number_in_suite" ]]; then
+        test_number_in_suite=$line_number
       fi
     fi
     ((++line_number))
   done <"$TESTS_FILE"
 
-  test_number_in_suite="$first_test_number_in_suite"
-  test_number_in_file=0
+  declare -rig BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE="$test_number_in_suite"
 }
 
 bats_run_tests() {
@@ -247,6 +271,8 @@ bats_run_tests() {
     export BATS_SEMAPHORE_NUMBER_OF_SLOTS="$num_jobs"
     bats_run_tests_in_parallel "$BATS_RUN_TMPDIR/parallel_output" || bats_exec_file_status=1
   else
+    local test_number_in_suite=$BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE \
+          test_number_in_file=0
     for test_name in "${tests_to_run[@]}"; do
       if [[ "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
         bats_exec_file_status=130 # bash's code for SIGINT exits
@@ -256,7 +282,8 @@ bats_run_tests() {
       if [[ $test_name ]]; then
         ((++test_number_in_suite))
         ((++test_number_in_file))
-        "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1
+        bats_run_test_with_retries "${flags[@]}" "$filename" "$test_name" \
+                                    "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=$?
       fi
     done
   fi
@@ -268,7 +295,7 @@ bats_create_file_tempdirs() {
     printf 'Failed to create %s\n' "$bats_files_tmpdir" >&2
     exit 1
   fi
-  BATS_FILE_TMPDIR="$bats_files_tmpdir/$first_test_number_in_suite"
+  BATS_FILE_TMPDIR="$bats_files_tmpdir/${BATS_FILE_FIRST_TEST_NUMBER_IN_SUITE?}"
   if ! mkdir "$BATS_FILE_TMPDIR"; then
     printf 'Failed to create BATS_FILE_TMPDIR=%s\n' "$BATS_FILE_TMPDIR" >&2
     exit 1

libexec/bats-core/bats-exec-test

@@ -49,6 +49,7 @@ export BATS_TEST_FILENAME="$1"
 export BATS_TEST_NAME="$2"
 export BATS_SUITE_TEST_NUMBER="$3"
 export BATS_TEST_NUMBER="$4"
+BATS_TEST_TRY_NUMBER="$5"
 
 if [[ -z "$BATS_TEST_FILENAME" ]]; then
   printf 'usage: bats-exec-test <filename>\n' >&2
@@ -67,7 +68,7 @@ bats_create_test_tmpdirs() {
 
   BATS_TEST_TMPDIR="$tests_tmpdir/$BATS_SUITE_TEST_NUMBER"
   if ! mkdir "$BATS_TEST_TMPDIR"; then
-    printf 'Failed to create BATS_TEST_TMPDIR: %s\n' "$BATS_TEST_TMPDIR" >&2
+    printf 'Failed to create BATS_TEST_TMPDIR%d: %s\n' "$BATS_TEST_TRY_NUMBER" "$BATS_TEST_TMPDIR" >&2
     exit 1
   fi
 
@@ -123,6 +124,7 @@ bats_exit_trap() {
 
   local print_bats_out="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS}"
 
+  local should_retry=''
   if [[ -z "$BATS_TEST_COMPLETED" || -z "$BATS_TEARDOWN_COMPLETED" || "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
     if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then
       # For some versions of bash, `$?` may not be set properly for some error
@@ -136,36 +138,46 @@ bats_exit_trap() {
       # output, since there's no way to reach the `bats_exit_trap` call.
       BATS_ERROR_STATUS=1
     fi
-    printf 'not ok %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3
-    local stack_trace
-    bats_get_failure_stack_trace stack_trace
-    bats_print_stack_trace "${stack_trace[@]}" >&3
-    bats_print_failed_command "${stack_trace[@]}" >&3
-
-    if [[ $BATS_PRINT_OUTPUT_ON_FAILURE && -n "${output:-}" ]]; then
-      printf "Last output:\n%s\n" "$output" >> "$BATS_OUT"
-    fi 
-
-    print_bats_out=1
-    status=1
-    local state=failed
+    if bats_should_retry_test; then
+      should_retry=1
+      status=126 # signify retry
+      rm -r "$BATS_TEST_TMPDIR" # clean up for retry
+    else
+      printf 'not ok %d %s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3
+      local stack_trace
+      bats_get_failure_stack_trace stack_trace
+      bats_print_stack_trace "${stack_trace[@]}" >&3
+      bats_print_failed_command "${stack_trace[@]}" >&3
+
+      if [[ $BATS_PRINT_OUTPUT_ON_FAILURE && -n "${output:-}" ]]; then
+        printf "Last output:\n%s\n" "$output" >> "$BATS_OUT"
+      fi 
+
+      print_bats_out=1    
+      status=1
+      local state=failed
+    fi
   else
     printf 'ok %d %s%s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" \
       "$skipped" >&3
     status=0
     local state=passed
   fi
 
-  printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "$BATS_RUNLOG_FILE"
+  if [[ -z "$should_retry" ]]; then
+    printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "$BATS_RUNLOG_FILE"
 
-  if [[ $print_bats_out ]]; then
-    bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3
+    if [[ $print_bats_out ]]; then
+      bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3
+    fi
   fi
-
   if [[ $BATS_GATHER_TEST_OUTPUTS_IN ]]; then
-    cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER-$BATS_TEST_DESCRIPTION.log"
+    local try_suffix=
+    if [[ -n "$should_retry" ]]; then
+      try_suffix="-try$BATS_TEST_TRY_NUMBER"
+    fi
+    cp "$BATS_OUT" "$BATS_GATHER_TEST_OUTPUTS_IN/$BATS_SUITE_TEST_NUMBER$try_suffix-$BATS_TEST_DESCRIPTION.log"
   fi
-
   rm -f "$BATS_OUT"
   exit "$status"
 }

test/bats.bats

@@ -1364,3 +1364,46 @@ enforce_own_process_group() {
   run find .bats/run-logs -name '*.log'
   [ "$first_run_logs" == "$output" ]
 }
+
+@test "BATS_TEST_RETRIES allows for retrying tests" {
+  export LOG="$BATS_TEST_TMPDIR/call.log"
+  bats_require_minimum_version 1.5.0
+  run ! bats "$FIXTURE_ROOT/retry.bats"
+  [ "${lines[0]}" == '1..3' ]
+  [ "${lines[1]}" == 'not ok 1 Fail all' ]
+  [ "${lines[4]}" == 'ok 2 Fail once' ]
+  [ "${lines[5]}" == 'not ok 3 Override retries' ]
+
+  run cat "$LOG"
+  [ "${lines[0]}" == 'setup_file ' ] # should only be executed once
+
+  # 3x Fail All (give up after 3 tries/2 retries)
+  [ "${lines[1]}" == 'setup 1' ] # should be executed for each try
+  [ "${lines[2]}" == 'test_Fail_all 1' ]
+  [ "${lines[3]}" == 'teardown 1' ] # should be executed for each try
+  [ "${lines[4]}" == 'setup 2' ]
+  [ "${lines[5]}" == 'test_Fail_all 2' ]
+  [ "${lines[6]}" == 'teardown 2' ]
+  [ "${lines[7]}" == 'setup 3' ]
+  [ "${lines[8]}" == 'test_Fail_all 3' ]
+  [ "${lines[9]}" == 'teardown 3' ]
+
+  # 2x Fail once (pass second try/first retry)
+  [ "${lines[10]}" == 'setup 1' ]
+  [ "${lines[11]}" == 'test_Fail_once 1' ]
+  [ "${lines[12]}" == 'teardown 1' ]
+  [ "${lines[13]}" == 'setup 2' ]
+  [ "${lines[14]}" == 'test_Fail_once 2' ]
+  [ "${lines[15]}" == 'teardown 2' ]
+
+  # 2x Override retries (give up after second try/first retry)
+  [ "${lines[16]}" == 'setup 1' ]
+  [ "${lines[17]}" == 'test_Override_retries 1' ]
+  [ "${lines[18]}" == 'teardown 1' ]
+  [ "${lines[19]}" == 'setup 2' ]
+  [ "${lines[20]}" == 'test_Override_retries 2' ]
+  [ "${lines[21]}" == 'teardown 2' ]
+
+  [ "${lines[22]}" == 'teardown_file ' ] # should only be executed once
+
+}

test/fixtures/bats/retry.bats

@@ -0,0 +1,37 @@
+BATS_TEST_RETRIES=2 # means three tries per test
+
+log_caller() {
+    printf "%s %s\n" "${FUNCNAME[1]}" "$BATS_TEST_TRY_NUMBER"  >> "${LOG?}"
+}
+
+setup_file() {
+    log_caller
+}
+
+teardown_file() {
+    log_caller
+}
+
+setup() {
+    log_caller
+}
+
+teardown() {
+    log_caller
+}
+
+@test "Fail all" {
+    log_caller
+    false
+}
+
+@test "Fail once" {
+    log_caller
+    (( $BATS_TEST_TRY_NUMBER > 1 )) || false
+}
+
+@test "Override retries" {
+    log_caller
+    BATS_TEST_RETRIES=1
+    (( $BATS_TEST_TRY_NUMBER > 2 )) || false
+}