Merge pull request #483 from martin-schulze-vireso/feature/rerun_failed

Add --filter-status failed for running only failed tests from the last completed run

.gitignore

@@ -5,3 +5,4 @@
 /bats-*.tgz
 # we don't have any deps; un-ignore if that changes
 /package-lock.json
+test/.bats/run-logs/

docs/CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog][kac] and this project adheres to
 
 * using external formatters via `--formatter <absolute path>` (also works for
   `--report-formatter`) (#602)
+* running only tests that failed in the last run via `--filter-status failed` (#483)
 
 #### Documentation
 

lib/bats-core/common.bash

@@ -63,3 +63,36 @@ bats_require_minimum_version() { # <required version>
     BATS_GUARANTEED_MINIMUM_VERSION="$required_minimum_version"
   fi
 }
+
+bats_binary_search() { # <search-value> <array-name>
+  local -r search_value=$1 array_name=$2
+
+  if [[ $# -ne 2 ]]; then
+    printf "ERROR: bats_binary_search requires exactly 2 arguments: <search value> <array name>\n" >&2
+    return 2
+  fi
+
+  # we'd like to test if array is set but we cannot distinguish unset from empty arrays, so we need to skip that
+
+  local start=0 mid end mid_value 
+  # start is inclusive, end is exclusive ...
+  eval "end=\${#${array_name}[@]}"
+
+  # so start == end means empty search space
+  while (( start < end )); do
+    mid=$(( (start + end) / 2 ))
+    eval "mid_value=\${${array_name}[$mid]}"
+    if [[ "$mid_value" == "$search_value" ]]; then
+      return 0
+    elif [[ "$mid_value" < "$search_value" ]]; then
+      # This branch excludes equality -> +1 to skip the mid element.
+      # This +1 also avoids endless recursion on odd sized search ranges.
+      start=$((mid + 1))
+    else
+      end=$mid
+    fi
+  done
+
+  # did not find it -> its not there
+  return 1
+}

libexec/bats-core/bats

@@ -41,6 +41,10 @@ HELP_TEXT_HEADER
                             or 'custom' which requires setting $BATS_BEGIN_CODE_QUOTE and 
                             $BATS_END_CODE_QUOTE. Can also be set via $BATS_CODE_QUOTE_STYLE
   -f, --filter <regex>      Only run tests that match the regular expression
+  --filter-status <status>  Only run tests with the given status in the last completed (no CTRL+C/SIGINT) run.
+                            Valid <status> values are:
+                              failed - runs tests that failed or were not present in the last run
+                              missed - runs tests that were not present in the last run
   -F, --formatter <type>    Switch between formatters: pretty (default),
                               tap (default w/o term), tap13, junit, /<absolute path to formatter>
   --gather-test-outputs-in <directory>
@@ -246,6 +250,10 @@ while [[ "$#" -ne 0 ]]; do
     shift
     BATS_CODE_QUOTE_STYLE="$1"
   ;;
+  --filter-status)
+    shift
+    flags+=('--filter-status' "$1")
+    ;;
   -*)
     abort "Bad command line option '$1'"
     ;;

libexec/bats-core/bats-exec-file

@@ -3,19 +3,11 @@ set -eET
 
 flags=('--dummy-flag')
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
-filter=''
 extended_syntax=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
-  -c) ;;
-
-  -f)
-    shift
-    filter="$1"
-    flags+=('-f' "$filter")
-    ;;
   -j)
     shift
     num_jobs="$1"
@@ -264,12 +256,7 @@ bats_run_tests() {
       if [[ $test_name ]]; then
         ((++test_number_in_suite))
         ((++test_number_in_file))
-        # deal with empty flags to avoid spurious "unbound variable" errors on Bash 4.3 and lower
-        if [[ "${#flags[@]}" -gt 0 ]]; then
-          "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1
-        else
-          "$BATS_LIBEXEC/bats-exec-test" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1
-        fi
+        "$BATS_LIBEXEC/bats-exec-test" "${flags[@]}" "$filename" "$test_name" "$test_number_in_suite" "$test_number_in_file" || bats_exec_file_status=1
       fi
     done
   fi

libexec/bats-core/bats-exec-suite

@@ -6,6 +6,7 @@ filter=''
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 bats_no_parallelize_across_files=${BATS_NO_PARALLELIZE_ACROSS_FILES-}
 bats_no_parallelize_within_files=
+filter_status=''
 flags=('--dummy-flag') # add a dummy flag to prevent unset varialeb errors on empty array expansion in old bash versions
 setup_suite_file=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
@@ -23,7 +24,6 @@ while [[ "$#" -ne 0 ]]; do
   -f)
     shift
     filter="$1"
-    flags+=('-f' "$filter")
     ;;
   -j)
     shift
@@ -43,6 +43,10 @@ while [[ "$#" -ne 0 ]]; do
     bats_no_parallelize_within_files=1
     flags+=("--no-parallelize-within-files")
     ;;
+  --filter-status)
+    shift
+    filter_status="$1"
+    ;;
   --dummy-flag)
     ;;
   --trace)
@@ -86,37 +90,141 @@ fi
 # create a file that contains all (filtered) tests to run from all files
 TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt"
 
-all_tests=()
-for filename in "$@"; do
-  if [[ ! -f "$filename" ]]; then
-    abort "Test file \"${filename}\" does not exist"
-  fi
-
-  test_names=()
-  test_dupes=()
-  while read -r line; do
-    if [[ ! "$line" =~ ^bats_test_function\  ]]; then
-      continue
+bats_gather_tests() {
+  all_tests=()
+  for filename in "$@"; do
+    if [[ ! -f "$filename" ]]; then
+      abort "Test file \"${filename}\" does not exist"
     fi
-    line="${line%$'\r'}"
-    line="${line#* }"
-    test_line=$(printf "%s\t%s" "$filename" "$line")
-    all_tests+=("$test_line")
-    printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE"
-    # avoid unbound variable errors on empty array expansion with old bash versions
-    if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then
-      test_dupes+=("$line")
-      continue
+
+    test_names=()
+    test_dupes=()
+    while read -r line; do
+      if [[ ! "$line" =~ ^bats_test_function\  ]]; then
+        continue
+      fi
+      line="${line%$'\r'}"
+      line="${line#* }"
+      test_line=$(printf "%s\t%s" "$filename" "$line")
+      all_tests+=("$test_line")
+      printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE"
+      # avoid unbound variable errors on empty array expansion with old bash versions
+      if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then
+        test_dupes+=("$line")
+        continue
+      fi
+      test_names+=("$line")
+    done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")
+
+    if [[ "${#test_dupes[@]}" -ne 0 ]]; then
+      abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}"
     fi
-    test_names+=("$line")
-  done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")
+  done
 
-  if [[ "${#test_dupes[@]}" -ne 0 ]]; then
-    abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}"
+  test_count="${#all_tests[@]}"
+}
+
+TEST_ROOT=${1%/*}
+BATS_RUN_LOGS_DIRECTORY="$TEST_ROOT/.bats/run-logs"
+if [[ ! -d "$BATS_RUN_LOGS_DIRECTORY" ]]; then
+  if [[ -n "$filter_status" ]]; then
+    printf "Error: --filter-status needs '%s/' to save failed tests. Please create this folder, add it to .gitignore and try again.\n" "$BATS_RUN_LOGS_DIRECTORY"
+    exit 1
+  else
+    BATS_RUN_LOGS_DIRECTORY=
   fi
-done
+  # discard via sink instead of having a conditional later
+  export BATS_RUNLOG_FILE='/dev/null'
+else
+  # use UTC (-u) to avoid problems with TZ changes
+  BATS_RUNLOG_DATE=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
+  export BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}.log"
+fi
+
+bats_gather_tests "$@"
 
-test_count="${#all_tests[@]}"
+if [[ -n "$filter_status" ]]; then  
+  # shellcheck source=lib/bats-core/common.bash
+  source "$BATS_ROOT/lib/bats-core/common.bash"
+  case "$filter_status" in
+    failed)
+      bats_filter_test_by_status() { # <line>
+        ! bats_binary_search "$1" "passed_tests"
+      }
+    ;;
+    passed)
+      bats_filter_test_by_status() {
+        ! bats_binary_search "$1" "failed_tests"
+      }
+    ;;
+    missed)
+      bats_filter_test_by_status() {
+        ! bats_binary_search "$1" "failed_tests" && ! bats_binary_search "$1" "passed_tests" 
+      }
+    ;;
+    *)
+      printf "Error: Unknown value '%s' for --filter-status. Valid values are 'failed' and 'missed'.\n" "$filter_status">&2
+      exit 1
+    ;;
+  esac
+
+  if IFS='' read -d $'\n' -r BATS_PREVIOUS_RUNLOG_FILE < <(ls -1r "$BATS_RUN_LOGS_DIRECTORY"); then
+    BATS_PREVIOUS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/$BATS_PREVIOUS_RUNLOG_FILE"
+    if [[ $BATS_PREVIOUS_RUNLOG_FILE == "$BATS_RUNLOG_FILE" ]]; then
+      count=$(find "$BATS_RUN_LOGS_DIRECTORY" -name "$BATS_RUNLOG_DATE*" | wc -l)
+      BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}-$count.log"
+    fi
+    failed_tests=()
+    passed_tests=()
+    # store tests that were already filtered out in the last run for the same filter reason
+    last_filtered_tests=()
+    i=0
+    while read -rd $'\n' line; do
+      ((++i))
+      case "$line" in
+        "passed "*)
+          passed_tests+=("${line#passed }")
+        ;;
+        "failed "*)
+          failed_tests+=("${line#failed }")
+        ;;
+        "status-filtered $filter_status"*) # pick up tests that were filtered in the last round for the same status
+          last_filtered_tests+=("${line#status-filtered "$filter_status" }")
+        ;;
+        "status-filtered "*) # ignore other status-filtered lines
+        ;;
+        "#"*) # allow for comments
+        ;;
+        *)
+          printf "Error: %s:%d: Invalid format: %s\n" "$BATS_PREVIOUS_RUNLOG_FILE" "$i" "$line" >&2
+          exit 1
+        ;;
+      esac
+    done < <(sort "$BATS_PREVIOUS_RUNLOG_FILE")
+
+    filtered_tests=()
+    for line in "${all_tests[@]}"; do
+      if bats_filter_test_by_status "$line" && ! bats_binary_search "$line" last_filtered_tests; then
+        printf "%s\n" "$line"
+        filtered_tests+=("$line")
+      else
+        printf "status-filtered %s %s\n" "$filter_status" "$line" >> "$BATS_RUNLOG_FILE"
+      fi
+    done > "$TESTS_LIST_FILE"
+
+    # save filtered tests to exclude them again in next round
+    for test_line in "${last_filtered_tests[@]}"; do 
+      printf "status-filtered %s %s\n" "$filter_status" "$test_line"
+    done >> "$BATS_RUNLOG_FILE"
+
+    test_count="${#filtered_tests[@]}"
+    if [[ ${#failed_tests} -eq 0 && ${#filtered_tests[@]} -eq 0 ]]; then
+      printf "There where no failed tests in the last recorded run.\n" >&2
+    fi
+  else
+    printf "No recording of previous runs found. Running all tests!\n" >&2
+  fi
+fi
 
 if [[ -n "$count_only_flag" ]]; then
   printf '%d\n' "${test_count}"
@@ -178,6 +286,11 @@ bats_suite_exit_trap() {
   if [[ ${BATS_INTERRUPTED-NOTSET} != NOTSET ]]; then
     printf "\n# Received SIGINT, aborting ...\n\n"
   fi
+
+  if [[ -d "$BATS_RUN_LOGS_DIRECTORY" && -n "${BATS_INTERRUPTED}" ]]; then
+    # aborting a test run with CTRL+C does not save the runlog file
+    rm "$BATS_RUNLOG_FILE"
+  fi
   exit "$bats_exec_suite_status"
 }
 

libexec/bats-core/bats-exec-test

@@ -2,8 +2,6 @@
 set -eET
 
 # Variables used in other scripts.
-BATS_COUNT_ONLY=''
-BATS_TEST_FILTER=''
 BATS_ENABLE_TIMING=''
 BATS_EXTENDED_SYNTAX=''
 BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"
@@ -15,15 +13,6 @@ BATS_TEST_NAME_PREFIX="${BATS_TEST_NAME_PREFIX:-}"
 
 while [[ "$#" -ne 0 ]]; do
   case "$1" in
-  -c)
-    # shellcheck disable=SC2034
-    BATS_COUNT_ONLY=1
-    ;;
-  -f)
-    shift
-    # shellcheck disable=SC2034
-    BATS_TEST_FILTER="$1"
-    ;;
   -T)
     BATS_ENABLE_TIMING='-T'
     ;;
@@ -159,12 +148,16 @@ bats_exit_trap() {
 
     print_bats_out=1
     status=1
+    local state=failed
   else
     printf 'ok %d %s%s\n' "$BATS_SUITE_TEST_NUMBER" "${BATS_TEST_NAME_PREFIX:-}${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" \
       "$skipped" >&3
     status=0
+    local state=passed
   fi
 
+  printf "%s %s\t%s\n" "$state" "$BATS_TEST_FILENAME" "$BATS_TEST_NAME" >> "$BATS_RUNLOG_FILE"
+
   if [[ $print_bats_out ]]; then
     bats_prefix_lines_for_tap_output < "$BATS_OUT" | bats_replace_filename >&3
   fi

man/bats.1.ronn

@@ -59,6 +59,11 @@ OPTIONS
   * `-F`, `--formatter <type>`:
     Switch between formatters: pretty (default), tap (default w/o term), tap13, junit,
     `/<absolute path to formatter>`
+  * `--filter-status <status>`
+    Only run tests with the given status in the last completed (no CTRL+C/SIGINT) run.
+    Valid <status> values are:
+      failed - runs tests that failed or were not present in the last run
+      missed - runs tests that were not present in the last run
   * `--gather-test-outputs-in <directory>`:
     Gather the output of failing *and* passing tests as files in directory
   * `-h`, `--help`:

shellcheck.sh

@@ -7,7 +7,7 @@ while IFS=  read -r -d $'\0'; do
     targets+=("$REPLY")
 done < <(
   find . -type f \( -name \*.bash -o -name \*.sh \) -print0; \
-  find . -name '*.bats' -not -name '*_no_shellcheck*' -print0; \
+  find . -type f -name '*.bats' -not -name '*_no_shellcheck*' -print0; \
   find libexec -type f -print0;
   find bin -type f -print0)