Add --rerun-failed

libexec/bats-core/bats

@@ -228,6 +228,9 @@ while [[ "$#" -ne 0 ]]; do
     shift
     BATS_CODE_QUOTE_STYLE="$1"
   ;;
+  --rerun-failed)
+    flags+=('--rerun-failed')
+    ;;
   -*)
     abort "Bad command line option '$1'"
     ;;

libexec/bats-core/bats-exec-file

@@ -6,6 +6,7 @@ num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 filter=''
 extended_syntax=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
+bats_rerun_failed=''
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
@@ -31,6 +32,9 @@ while [[ "$#" -ne 0 ]]; do
     # use singular to allow for users to override in file
     BATS_NO_PARALLELIZE_WITHIN_FILE=1
     ;;
+  --rerun-failed)
+    flags+=(--rerun-failed)
+    ;;
   --dummy-flag)
     ;;
   --trace)

libexec/bats-core/bats-exec-suite

@@ -6,6 +6,7 @@ filter=''
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 bats_no_parallelize_across_files=${BATS_NO_PARALLELIZE_ACROSS_FILES-}
 bats_no_parallelize_within_files=
+bats_rerun_failed=
 flags=('--dummy-flag') # add a dummy flag to prevent unset varialeb errors on empty array expansion in old bash versions
 
 abort() {
@@ -41,6 +42,10 @@ while [[ "$#" -ne 0 ]]; do
     bats_no_parallelize_within_files=1
     flags+=("--no-parallelize-within-files")
     ;;
+  --rerun-failed)
+    bats_rerun_failed=1
+    flags+=("--rerun-failed")
+    ;;
   --dummy-flag)
     ;;
   --trace)
@@ -79,37 +84,59 @@ fi
 # create a file that contains all (filtered) tests to run from all files
 TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt"
 
-all_tests=()
-for filename in "$@"; do
-  if [[ ! -f "$filename" ]]; then
-    abort "Test file \"${filename}\" does not exist"
-  fi
-
-  test_names=()
-  test_dupes=()
-  while read -r line; do
-    if [[ ! "$line" =~ ^bats_test_function\  ]]; then
-      continue
+bats_gather_tests() {
+  all_tests=()
+  for filename in "$@"; do
+    if [[ ! -f "$filename" ]]; then
+      abort "Test file \"${filename}\" does not exist"
     fi
-    line="${line%$'\r'}"
-    line="${line#* }"
-    test_line=$(printf "%s\t%s" "$filename" "$line")
-    all_tests+=("$test_line")
-    printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE"
-    # avoid unbound variable errors on empty array expansion with old bash versions
-    if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then
-      test_dupes+=("$line")
-      continue
+
+    test_names=()
+    test_dupes=()
+    while read -r line; do
+      if [[ ! "$line" =~ ^bats_test_function\  ]]; then
+        continue
+      fi
+      line="${line%$'\r'}"
+      line="${line#* }"
+      test_line=$(printf "%s\t%s" "$filename" "$line")
+      all_tests+=("$test_line")
+      printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE"
+      # avoid unbound variable errors on empty array expansion with old bash versions
+      if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then
+        test_dupes+=("$line")
+        continue
+      fi
+      test_names+=("$line")
+    done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")
+
+    if [[ "${#test_dupes[@]}" -ne 0 ]]; then
+      abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}"
     fi
-    test_names+=("$line")
-  done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")
+  done
 
-  if [[ "${#test_dupes[@]}" -ne 0 ]]; then
-    abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}"
-  fi
-done
+  test_count="${#all_tests[@]}"
+}
 
-test_count="${#all_tests[@]}"
+if [[ -n "$bats_rerun_failed" ]]; then  
+  mkdir -p "$PWD/.bats"
+  export BATS_RERUN_FAILED_FILE="$PWD/.bats/rerun-failed-tests.list"
+  export BATS_RERUN_FAILED_TMPFILE="${BATS_RERUN_FAILED_FILE}.tmp"
+  if [[ -s "$BATS_RERUN_FAILED_FILE" ]]; then
+    test_count="$(wc -l < "$BATS_RERUN_FAILED_FILE")"
+    cp "$BATS_RERUN_FAILED_FILE" "$TESTS_LIST_FILE"
+  elif [[ -f "$BATS_RERUN_FAILED_FILE" ]]; then
+    printf "There where no failed tests in the last recorded run.\n" >&2
+    printf "Delete the file '%s' to run all tests again.\n" "${BATS_RERUN_FAILED_FILE#"$PWD/"}" >&2
+    test_count=0
+  else
+    printf "No recording of previos runs found. Running all tests!\n" >&2
+    bats_gather_tests "$@"
+  fi 
+  : >"$BATS_RERUN_FAILED_TMPFILE" # truncate the file if it exists, else create
+else
+  bats_gather_tests "$@"
+fi
 
 if [[ -n "$count_only_flag" ]]; then
   printf '%d\n' "${test_count}"
@@ -174,4 +201,10 @@ else
   done
 fi
 
+if [[ -n "$bats_rerun_failed" && "${BATS_INTERRUPTED-NOTSET}" == NOTSET ]]; then
+  # only overwrite file once we finished with all tests
+  # else we would discard tests when aborting a test run with CTRL+C
+  mv "$BATS_RERUN_FAILED_TMPFILE" "$BATS_RERUN_FAILED_FILE"
+fi
+
 exit "$status"

libexec/bats-core/bats-exec-test

@@ -11,6 +11,7 @@ BATS_PRINT_OUTPUT_ON_FAILURE="${BATS_PRINT_OUTPUT_ON_FAILURE:-}"
 BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS:-}"
 BATS_VERBOSE_RUN="${BATS_VERBOSE_RUN:-}"
 BATS_GATHER_TEST_OUTPUTS_IN="${BATS_GATHER_TEST_OUTPUTS_IN:-}"
+bats_rerun_failed=
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
@@ -48,6 +49,9 @@ while [[ "$#" -ne 0 ]]; do
     shift
     BATS_GATHER_TEST_OUTPUTS_IN="$1"
     ;;
+  --rerun-failed)
+    bats_rerun_failed=1
+    ;;
   *)
     break
     ;;
@@ -134,6 +138,9 @@ bats_exit_trap() {
   local print_bats_out="${BATS_SHOW_OUTPUT_OF_SUCCEEDING_TESTS}"
 
   if [[ -z "$BATS_TEST_COMPLETED" || -z "$BATS_TEARDOWN_COMPLETED" || "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
+    if [[ -n "$bats_rerun_failed" ]]; then
+      printf "%s\t%s\n" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "${BATS_RERUN_FAILED_TMPFILE}"
+    fi
     if [[ "$BATS_ERROR_STATUS" -eq 0 ]]; then
       # For some versions of bash, `$?` may not be set properly for some error
       # conditions before triggering the EXIT trap directly (see #72 and #81).

test/bats.bats

@@ -1259,3 +1259,62 @@ EOF
     [[ $FDS_LOG == *'otherfunc fds after: (0 1 2)'* ]] || false
     [[ $FDS_LOG == *'setup_file fds after: (0 1 2)'* ]] || false
 }
+
+@test "Without previous recording --rerun-failed runs all tests and then reruns only failed tests" {
+  cd "$BATS_TEST_TMPDIR" # don't pollute the source folder
+  run -1 bats --rerun-failed "$FIXTURE_ROOT/passing_and_failing.bats"
+  # without previous recording, all tests should be run
+  [ "${lines[0]}" == 'No recording of previos runs found. Running all tests!' ]
+  [ "${lines[1]}" == '1..2' ]
+  [ "${lines[2]}" == 'ok 1 a passing test' ]
+  [ "${lines[3]}" == 'not ok 2 a failing test' ]
+
+  run -1 bats --tap --rerun-failed "$FIXTURE_ROOT/passing_and_failing.bats"
+  # now we should only run the failing test
+  [ "${lines[0]}" == 1..1 ]
+  [ "${lines[1]}" == "not ok 1 a failing test" ]
+}
+
+@test "--rerun-failed gives warning on empty failed test list" {
+  cd "$BATS_TEST_TMPDIR" # don't pollute the source folder
+  # have no failing tests
+  run -0 bats --rerun-failed "$FIXTURE_ROOT/passing.bats"
+  # try to rerun the empty list of failing tests
+  run -0 bats --rerun-failed "$FIXTURE_ROOT/passing.bats"
+  [ "${lines[0]}" == "There where no failed tests in the last recorded run." ]
+  [ "${lines[1]}" == "Delete the file '.bats/rerun-failed-tests.list' to run all tests again." ]
+  [ "${lines[2]}" == "1..0" ]
+  [ "${#lines[@]}" -eq 3 ]
+}
+
+enforce_own_process_group() {
+    set -m
+    "$@"
+  }
+
+@test "--rerun-failed does not update list when run is aborted" {
+  if [[ "$BATS_NUMBER_OF_PARALLEL_JOBS" -gt 1 ]]; then
+    skip "Aborts don't work in parallel mode"
+  fi
+
+  cd "$BATS_TEST_TMPDIR" # don't pollute the source folder
+  local RERUN_FILE=".bats/rerun-failed-tests.list"
+
+  # don't hang yet, so we get a useful rerun file
+  run -1 env bats --rerun-failed "$FIXTURE_ROOT/sigint_in_failing_test.bats"
+
+  orig_date=$(date -r "$RERUN_FILE")
+
+  # check that we have something to rerun
+  [ -s "$RERUN_FILE" ]
+
+  sleep 1 # ensure we would get different timestamps for each run
+
+  # now rerun but abort midrun
+  run -1 enforce_own_process_group bats --rerun-failed "$FIXTURE_ROOT/sigint_in_failing_test.bats"
+
+  new_date=$(date -r "$RERUN_FILE")
+  echo "$orig_date"
+  echo "$new_date"
+  [ "$orig_date" == "$new_date" ]
+}

test/fixtures/bats/sigint_in_failing_test.bats

@@ -0,0 +1,15 @@
+setup() {
+    load '../../concurrent-coordination'
+}
+
+@test "failing" {
+    if [[ -z "${DONT_ABORT:-}" ]]; then
+        # emulate CTRL-C by sending SIGINT to the whole process group
+        kill -SIGINT -- -$BATS_ROOT_PID
+    fi
+    false
+}
+
+@test "passing" {
+    :
+}