bats-exec-suite

#!/usr/bin/env bash
set -e

count_only_flag=''
filter=''
num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
bats_no_parallelize_across_files=${BATS_NO_PARALLELIZE_ACROSS_FILES-}
bats_no_parallelize_within_files=
filter_status=''
flags=('--dummy-flag') # add a dummy flag to prevent unset variable errors on empty array expansion in old bash versions
setup_suite_file=''
BATS_TRACE_LEVEL="${BATS_TRACE_LEVEL:-0}"

abort() {
  printf 'Error: %s\n' "$1" >&2
  exit 1
}

while [[ "$#" -ne 0 ]]; do
  case "$1" in
  -c)
    count_only_flag=1
    ;;
  -f)
    shift
    filter="$1"
    ;;
  -j)
    shift
    num_jobs="$1"
    flags+=('-j' "$num_jobs")
    ;;
  -T)
    flags+=('-T')
    ;;
  -x)
    flags+=('-x')
    ;;
  --no-parallelize-across-files)
    bats_no_parallelize_across_files=1
    ;;
  --no-parallelize-within-files)
    bats_no_parallelize_within_files=1
    flags+=("--no-parallelize-within-files")
    ;;
  --filter-status)
    shift
    filter_status="$1"
    ;;
  --dummy-flag)
    ;;
  --trace)
    flags+=('--trace')
    (( ++BATS_TRACE_LEVEL )) # avoid returning 0
    ;;
  --print-output-on-failure)
    flags+=(--print-output-on-failure)
    ;;
  --show-output-of-passing-tests)
    flags+=(--show-output-of-passing-tests)
    ;;
  --verbose-run)
    flags+=(--verbose-run)
    ;;
  --gather-test-outputs-in)
    shift
    flags+=(--gather-test-outputs-in "$1")
    ;;
  --setup-suite-file)
    shift
    setup_suite_file="$1"
    ;;
  *)
    break
    ;;
  esac
  shift
done

if [[ "$num_jobs" != 1 ]]; then
  if ! type -p parallel >/dev/null && [[ -z "$bats_no_parallelize_across_files" ]]; then
    abort "Cannot execute \"${num_jobs}\" jobs without GNU parallel"
    exit 1
  fi
  # shellcheck source=lib/bats-core/semaphore.bash
  source "${BATS_ROOT}/lib/bats-core/semaphore.bash"
  bats_semaphore_setup
fi

# create a file that contains all (filtered) tests to run from all files
TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt"

bats_gather_tests() {
  all_tests=()
  for filename in "$@"; do
    if [[ ! -f "$filename" ]]; then
      abort "Test file \"${filename}\" does not exist"
    fi

    test_names=()
    test_dupes=()
    while read -r line; do
      if [[ ! "$line" =~ ^bats_test_function\  ]]; then
        continue
      fi
      line="${line%$'\r'}"
      line="${line#* }"
      test_line=$(printf "%s\t%s" "$filename" "$line")
      all_tests+=("$test_line")
      printf "%s\n" "$test_line" >>"$TESTS_LIST_FILE"
      # avoid unbound variable errors on empty array expansion with old bash versions
      if [[ ${#test_names[@]} -gt 0 && " ${test_names[*]} " == *" $line "* ]]; then
        test_dupes+=("$line")
        continue
      fi
      test_names+=("$line")
    done < <(BATS_TEST_FILTER="$filter" bats-preprocess "$filename")

    if [[ "${#test_dupes[@]}" -ne 0 ]]; then
      abort "Duplicate test name(s) in file \"${filename}\": ${test_dupes[*]}"
    fi
  done

  test_count="${#all_tests[@]}"
}

TEST_ROOT=${1-}
TEST_ROOT=${TEST_ROOT%/*}
BATS_RUN_LOGS_DIRECTORY="$TEST_ROOT/.bats/run-logs"
if [[ ! -d "$BATS_RUN_LOGS_DIRECTORY" ]]; then
  if [[ -n "$filter_status" ]]; then
    printf "Error: --filter-status needs '%s/' to save failed tests. Please create this folder, add it to .gitignore and try again.\n" "$BATS_RUN_LOGS_DIRECTORY"
    exit 1
  else
    BATS_RUN_LOGS_DIRECTORY=
  fi
  # discard via sink instead of having a conditional later
  export BATS_RUNLOG_FILE='/dev/null'
else
  # use UTC (-u) to avoid problems with TZ changes
  BATS_RUNLOG_DATE=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
  export BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}.log"
fi

bats_gather_tests "$@"

if [[ -n "$filter_status" ]]; then  
  # shellcheck source=lib/bats-core/common.bash
  source "$BATS_ROOT/lib/bats-core/common.bash"
  case "$filter_status" in
    failed)
      bats_filter_test_by_status() { # <line>
        ! bats_binary_search "$1" "passed_tests"
      }
    ;;
    passed)
      bats_filter_test_by_status() {
        ! bats_binary_search "$1" "failed_tests"
      }
    ;;
    missed)
      bats_filter_test_by_status() {
        ! bats_binary_search "$1" "failed_tests" && ! bats_binary_search "$1" "passed_tests" 
      }
    ;;
    *)
      printf "Error: Unknown value '%s' for --filter-status. Valid values are 'failed' and 'missed'.\n" "$filter_status">&2
      exit 1
    ;;
  esac

  if IFS='' read -d $'\n' -r BATS_PREVIOUS_RUNLOG_FILE < <(ls -1r "$BATS_RUN_LOGS_DIRECTORY"); then
    BATS_PREVIOUS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/$BATS_PREVIOUS_RUNLOG_FILE"
    if [[ $BATS_PREVIOUS_RUNLOG_FILE == "$BATS_RUNLOG_FILE" ]]; then
      count=$(find "$BATS_RUN_LOGS_DIRECTORY" -name "$BATS_RUNLOG_DATE*" | wc -l)
      BATS_RUNLOG_FILE="$BATS_RUN_LOGS_DIRECTORY/${BATS_RUNLOG_DATE}-$count.log"
    fi
    failed_tests=()
    passed_tests=()
    # store tests that were already filtered out in the last run for the same filter reason
    last_filtered_tests=()
    i=0
    while read -rd $'\n' line; do
      ((++i))
      case "$line" in
        "passed "*)
          passed_tests+=("${line#passed }")
        ;;
        "failed "*)
          failed_tests+=("${line#failed }")
        ;;
        "status-filtered $filter_status"*) # pick up tests that were filtered in the last round for the same status
          last_filtered_tests+=("${line#status-filtered "$filter_status" }")
        ;;
        "status-filtered "*) # ignore other status-filtered lines
        ;;
        "#"*) # allow for comments
        ;;
        *)
          printf "Error: %s:%d: Invalid format: %s\n" "$BATS_PREVIOUS_RUNLOG_FILE" "$i" "$line" >&2
          exit 1
        ;;
      esac
    done < <(sort "$BATS_PREVIOUS_RUNLOG_FILE")

    filtered_tests=()
    for line in "${all_tests[@]}"; do
      if bats_filter_test_by_status "$line" && ! bats_binary_search "$line" last_filtered_tests; then
        printf "%s\n" "$line"
        filtered_tests+=("$line")
      else
        printf "status-filtered %s %s\n" "$filter_status" "$line" >> "$BATS_RUNLOG_FILE"
      fi
    done > "$TESTS_LIST_FILE"

    # save filtered tests to exclude them again in next round
    for test_line in "${last_filtered_tests[@]}"; do 
      printf "status-filtered %s %s\n" "$filter_status" "$test_line"
    done >> "$BATS_RUNLOG_FILE"

    test_count="${#filtered_tests[@]}"
    if [[ ${#failed_tests[@]} -eq 0 && ${#filtered_tests[@]} -eq 0 ]]; then
      printf "There where no failed tests in the last recorded run.\n" >&2
    fi
  else
    printf "No recording of previous runs found. Running all tests!\n" >&2
  fi
fi

if [[ -n "$count_only_flag" ]]; then
  printf '%d\n' "${test_count}"
  exit
fi

if [[ -n "$bats_no_parallelize_across_files" ]] && [[ ! "$num_jobs" -gt 1 ]]; then
  abort "The flag --no-parallelize-across-files requires at least --jobs 2"
  exit 1
fi

if [[ -n "$bats_no_parallelize_within_files" ]] && [[ ! "$num_jobs" -gt 1 ]]; then
  abort "The flag --no-parallelize-across-files requires at least --jobs 2"
  exit 1
fi

# only abort on the lowest levels
trap 'BATS_INTERRUPTED=true' INT

bats_exec_suite_status=0
printf '1..%d\n' "${test_count}"

# No point on continuing if there's no tests.
if [[ "${test_count}" == 0 ]]; then
  exit
fi

export BATS_SUITE_TMPDIR="${BATS_RUN_TMPDIR}/suite"
if ! mkdir "$BATS_SUITE_TMPDIR"; then
  printf '%s\n' "Failed to create BATS_SUITE_TMPDIR" >&2
  exit 1
fi

# Deduplicate filenames (without reordering) to avoid running duplicate tests n by n times.
# (see https://github.com/bats-core/bats-core/issues/329)
# If a file was specified multiple times, we already got it repeatedly in our TESTS_LIST_FILE.
# Thus, it suffices to bats-exec-file it once to run all repeated tests on it.
IFS=$'\n' read -d '' -r -a  BATS_UNIQUE_TEST_FILENAMES < <(printf "%s\n" "$@"| nl | sort -k 2 | uniq -f 1 | sort -n | cut -f 2-) || true

# shellcheck source=lib/bats-core/tracing.bash
source "$BATS_ROOT/lib/bats-core/tracing.bash"
bats_setup_tracing

trap bats_suite_exit_trap EXIT

bats_suite_exit_trap() {
  if [[ -z "${BATS_SETUP_SUITE_COMPLETED}" || -z "${BATS_TEARDOWN_SUITE_COMPLETED}" ]]; then
    if [[ -z "${BATS_SETUP_SUITE_COMPLETED}" ]]; then
      printf "not ok 1 setup_suite\n"
    elif [[ -z "${BATS_TEARDOWN_SUITE_COMPLETED}" ]]; then
      printf "not ok %d teardown_suite\n" $((test_count+1))
    fi
    local stack_trace
    bats_get_failure_stack_trace stack_trace
    bats_print_stack_trace "${stack_trace[@]}"
    bats_print_failed_command "${stack_trace[@]}"
    bats_exec_suite_status=1
  fi
  if [[ ${BATS_INTERRUPTED-NOTSET} != NOTSET ]]; then
    printf "\n# Received SIGINT, aborting ...\n\n"
  fi
  
  if [[ -d "$BATS_RUN_LOGS_DIRECTORY" && -n "${BATS_INTERRUPTED:-}" ]]; then
    # aborting a test run with CTRL+C does not save the runlog file
    rm "$BATS_RUNLOG_FILE"
  fi
  exit "$bats_exec_suite_status"
}

bats_run_teardown_suite() {
  # avoid being called twice, in case this is not called through bats_teardown_suite_trap
  # but from the end of file
  trap bats_suite_exit_trap EXIT
  set -eET
  BATS_TEARDOWN_SUITE_COMPLETED=
  teardown_suite 2>&1
  BATS_TEARDOWN_SUITE_COMPLETED=1
  set +ET
}

bats_teardown_suite_trap() {
  bats_run_teardown_suite
  bats_suite_exit_trap
}

setup_suite() {
  :
}

teardown_suite() {
  : 
}

trap bats_teardown_suite_trap EXIT

if [[ -n "$setup_suite_file" ]]; then
  setup_suite() {
    printf "%s does not define \`setup_suite()\`\n" "$setup_suite_file" >&2
    exit 1
  }

  # shellcheck disable=SC1090
  source "$setup_suite_file"
fi

set -eET
BATS_SETUP_SUITE_COMPLETED=
setup_suite 2>&1
BATS_SETUP_SUITE_COMPLETED=1
set +ET

if [[ "$num_jobs" -gt 1 ]] && [[ -z "$bats_no_parallelize_across_files" ]]; then
  # run files in parallel to get the maximum pool of parallel tasks
  # shellcheck disable=SC2086,SC2068
  # we need to handle the quoting of ${flags[@]} ourselves,
  # because parallel can only quote it as one
  parallel --keep-order --jobs "$num_jobs" bats-exec-file "$(printf "%q " "${flags[@]}")" "{}" "$TESTS_LIST_FILE"  ::: "${BATS_UNIQUE_TEST_FILENAMES[@]}" 2>&1 || bats_exec_suite_status=1
else
  for filename in "${BATS_UNIQUE_TEST_FILENAMES[@]}"; do
    if [[ "${BATS_INTERRUPTED-NOTSET}" != NOTSET ]]; then
      bats_exec_suite_status=130 # bash's code for SIGINT exits
      break
    fi
    bats-exec-file "${flags[@]}" "$filename" "${TESTS_LIST_FILE}" || bats_exec_suite_status=1
  done
fi

set -eET
bats_run_teardown_suite

exit "$bats_exec_suite_status"