Skip to content

Commit

Permalink
Add timeout to all formatters, simplify interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@martin-schulze-vireso
martin-schulze-vireso committed Aug 12, 2022
1 parent 475326e commit b1e7fed
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 120 deletions.
46 changes: 27 additions & 19 deletions lib/bats-core/formatter.bash
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,25 @@
#!/usr/bin/env bash

# reads (extended) bats tap streams from stdin and calls callback functions for each line
#
# Segmenting functions
# ====================
# bats_tap_stream_plan <number of tests> -> when the test plan is encountered
# bats_tap_stream_suite <file name> -> when a new file is begun WARNING: extended only
# bats_tap_stream_begin <test index> <test name> -> when a new test is begun WARNING: extended only
# bats_tap_stream_ok [--duration <milliseconds] <test index> <test name> -> when a test was successful
# bats_tap_stream_not_ok [--duration <milliseconds>] <test index> <test name> [<timeout-in-seconds>]
# -> when a test has failed. also reports the timeout limit
# if the failure was due to timeout
# bats_tap_stream_skipped [--duraction <milliseconds>] <test index> <test name> <skip reason> -> when a test was skipped
#
# Test result functions
# =====================
# If timing was enabled, BATS_FORMATTER_TEST_DURATION will be set to their duration in milliseconds
# bats_tap_stream_ok <test index> <test name> -> when a test was successful
# bats_tap_stream_not_ok <test index> <test name> -> when a test has failed. If the failure was due to a timeout,
# BATS_FORMATTER_TEST_TIMEOUT is set to the timeout duration in seconds
# bats_tap_stream_skipped <test index> <test name> <skip reason> -> when a test was skipped
#
# Context functions
# =================
# bats_tap_stream_comment <comment text without leading '# '> <scope> -> when a comment line was encountered,
# scope tells the last encountered of plan, begin, ok, not_ok, skipped, suite
# bats_tap_stream_suite <file name> -> when a new file is begun WARNING: extended only
# bats_tap_stream_unknown <full line> <scope> -> when a line is encountered that does not match the previous entries,
# scope @see bats_tap_stream_comment
# forwards all input as is, when there is no TAP test plan header
Expand All @@ -37,6 +46,7 @@ function bats_parse_internal_extended_tap() {
index=0
scope=plan
while IFS= read -r line; do
unset BATS_FORMATTER_TEST_DURATION BATS_FORMATTER_TEST_TIMEOUT
case "$line" in
'begin '*) # this might only be called in extended tap output
((++begin_index))
Expand All @@ -54,16 +64,17 @@ function bats_parse_internal_extended_tap() {
test_name="${BASH_REMATCH[2]}" # cut off name before "# skip"
local skip_reason="${BASH_REMATCH[4]}"
if [[ "$test_name" =~ $timing_expr ]]; then
timing="${BASH_REMATCH[1]}"
test_name="${test_name% in "${timing}"ms}"
bats_tap_stream_skipped --duration "$timing" "$ok_index" "$test_name" "$skip_reason"
local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}"
test_name="${test_name% in "${BATS_FORMATTER_TEST_DURATION}"ms}"
bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason"
else
bats_tap_stream_skipped "$ok_index" "$test_name" "$skip_reason"
fi
else
scope=ok
if [[ "$line" =~ $timing_expr ]]; then
bats_tap_stream_ok --duration "${BASH_REMATCH[1]}" "$ok_index" "${test_name% in "${BASH_REMATCH[1]}"ms}"
local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}"
bats_tap_stream_ok "$ok_index" "${test_name% in "${BASH_REMATCH[1]}"ms}"
else
bats_tap_stream_ok "$ok_index" "$test_name"
fi
Expand All @@ -79,21 +90,18 @@ function bats_parse_internal_extended_tap() {
if [[ "$line" =~ $not_ok_line_regexpr ]]; then
not_ok_index="${BASH_REMATCH[1]}"
test_name="${BASH_REMATCH[2]}"
local options=() timeout=-1
if [[ "$line" =~ $timeout_line_regexpr ]]; then
not_ok_index="${BASH_REMATCH[1]}"
test_name="${BASH_REMATCH[2]}"
timeout="${BASH_REMATCH[3]}"
# shellcheck disable=SC2034 # used in bats_tap_stream_ok
local BATS_FORMATTER_TEST_TIMEOUT="${BASH_REMATCH[3]}"
fi
if [[ "$test_name" =~ $timing_expr ]]; then
options=(--duration "${BASH_REMATCH[1]}" "$not_ok_index" "${test_name% in "${BASH_REMATCH[1]}"ms}")
else
options=("$not_ok_index" "$test_name")
fi
if (( timeout != -1 )); then
options+=("$timeout")
# shellcheck disable=SC2034 # used in bats_tap_stream_ok
local BATS_FORMATTER_TEST_DURATION="${BASH_REMATCH[1]}"
test_name="${test_name% in "${BASH_REMATCH[1]}"ms}"
fi
bats_tap_stream_not_ok "${options[@]}"
bats_tap_stream_not_ok "$not_ok_index" "$test_name"
else
printf "ERROR: could not match not ok line: %s" "$line" >&2
exit 1
Expand Down
17 changes: 5 additions & 12 deletions libexec/bats-core/bats-format-junit
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,34 +186,27 @@ bats_tap_stream_begin() { # <test index> <test name>
name="$2"
}

bats_tap_stream_ok() { # [--duration <milliseconds] <test index> <test name>
if [[ "$1" == "--duration" ]]; then
test_exec_time="${BASH_REMATCH[1]}"
else
test_exec_time=0
fi
bats_tap_stream_ok() { # <test index> <test name>
test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0}
((file_count += 1))
test_result_state='ok'
file_exec_time="$((file_exec_time + test_exec_time))"
suite_test_exec_time=$((suite_test_exec_time + test_exec_time))
}

bats_tap_stream_skipped() { # <test index> <test name> <skip reason>
test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0}
((file_count += 1))
((file_skipped += 1))
test_result_state='skipped'
test_exec_time=0
test_skip_message="$3"
}

bats_tap_stream_not_ok() { # [--duration <milliseconds>] <test index> <test name>
bats_tap_stream_not_ok() { # <test index> <test name>
test_exec_time=${BATS_FORMATTER_TEST_DURATION:-0}
((file_count += 1))
((file_failures += 1))
if [[ "$1" == "--duration" ]]; then
test_exec_time="${BASH_REMATCH[1]}"
else
test_exec_time=0
fi
test_result_state=not_ok
file_exec_time="$((file_exec_time + test_exec_time))"
suite_test_exec_time=$((suite_test_exec_time + test_exec_time))
Expand Down
25 changes: 5 additions & 20 deletions libexec/bats-core/bats-format-pretty
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,44 +273,29 @@ bats_tap_stream_begin() {
}

bats_tap_stream_ok() {
local duration=
if [[ "$1" == "--duration" ]]; then
duration="$2"
shift 2
fi
index="$1"
name="$2"
((++passed))

pass "$duration"
pass "${BATS_FORMATTER_TEST_DURATION:-}"
}

bats_tap_stream_skipped() {
local duration=
if [[ "$1" == "--duration" ]]; then
duration="$2"
shift 2
fi
index="$1"
name="$2"
((++skipped))
skip "$3" "$duration"
skip "$3" "${BATS_FORMATTER_TEST_DURATION:-}"
}

bats_tap_stream_not_ok() {
local duration=
if [[ "$1" == "--duration" ]]; then
duration="$2"
shift 2
fi
index="$1"
name="$2"

if (( $# == 3 )); then
timeout "$duration" "$3s"
if [[ ${BATS_FORMATTER_TEST_TIMEOUT-x} != x ]]; then
timeout "${BATS_FORMATTER_TEST_DURATION:-}" "${BATS_FORMATTER_TEST_TIMEOUT}s"
((++timed_out))
else
fail "$duration"
fail "${BATS_FORMATTER_TEST_DURATION:-}"
((++failures))
fi

Expand Down
25 changes: 14 additions & 11 deletions libexec/bats-core/bats-format-tap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,27 @@ bats_tap_stream_begin() { #<test index> <test name>
:
}

bats_tap_stream_ok() { # [--duration <milliseconds] <test index> <test name>
if [[ "$1" == "--duration" ]]; then
printf "ok %d %s # in %d ms\n" "$3" "$4" "$2"
else
printf "ok %d %s\n" "$1" "$2"
bats_tap_stream_ok() { # [<test index> <test name>
printf "ok %d %s" "$1" "$2"
if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then
printf " # in %d ms" "$BATS_FORMATTER_TEST_DURATION"
fi
printf "\n"
}

bats_tap_stream_not_ok() { # [--duration <milliseconds>] <test index> <test name>
if [[ "$1" == "--duration" ]]; then
printf "not ok %d %s # in %d ms\n" "$3" "$4" "$2"
else
printf "not ok %d %s\n" "$1" "$2"
bats_tap_stream_not_ok() { # <test index> <test name>
printf "not ok %d %s" "$1" "$2"
if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then
printf " # in %d ms" "$BATS_FORMATTER_TEST_DURATION"
fi
if [[ "${BATS_FORMATTER_TEST_TIMEOUT-x}" != x ]]; then
printf " # timeout after %d s" "${BATS_FORMATTER_TEST_TIMEOUT}"
fi
printf "\n"
}

bats_tap_stream_skipped() { # <test index> <test name> <reason>
if [[ -n "$3" ]]; then
if [[ $# -eq 3 ]]; then
printf "ok %d %s # skip %s\n" "$1" "$2" "$3"
else
printf "ok %d %s # skip\n" "$1" "$2"
Expand Down
114 changes: 56 additions & 58 deletions libexec/bats-core/bats-format-tap13
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,6 @@
#!/usr/bin/env bash
set -e

while [[ "$#" -ne 0 ]]; do
case "$1" in
-T)
BATS_ENABLE_TIMING="-T"
;;
esac
shift
done

header_pattern='[0-9]+\.\.[0-9]+'
IFS= read -r header

if [[ "$header" =~ $header_pattern ]]; then
printf "TAP version 13\n"
printf "%s\n" "$header"
else
# If the first line isn't a TAP plan, print it and pass the rest through
printf '%s\n' "$header"
exec cat
fi

yaml_block_open=''
add_yaml_entry() {
if [[ -z "$yaml_block_open" ]]; then
Expand All @@ -42,50 +21,69 @@ trap '' INT

number_of_printed_log_lines_for_this_test_so_far=0

while IFS= read -r line; do
case "$line" in
'begin '*) ;;
'ok '*)
# shellcheck source=lib/bats-core/formatter.bash
source "$BATS_ROOT/lib/bats-core/formatter.bash"

bats_tap_stream_plan() {
printf "TAP version 13\n"
printf "1..%d\n" "$1"
}

bats_tap_stream_begin() { #<test index> <test name>
:
}

bats_tap_stream_ok() { # <test index> <test name>
close_previous_yaml_block
number_of_printed_log_lines_for_this_test_so_far=0
if [[ -n "${BATS_ENABLE_TIMING-}" ]]; then
timing_expr="(ok [0-9]+ .+) in ([0-9]+)ms$"
if [[ "$line" =~ $timing_expr ]]; then
printf "%s\n" "${BASH_REMATCH[1]}"
add_yaml_entry "duration_ms" "${BASH_REMATCH[2]}"
else
echo "Could not match output line to timing regex: $line" >&2
exit 1
fi
else
printf "%s\n" "${line}"
printf "ok %d %s\n" "$1" "$2"
if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then
add_yaml_entry "duration_ms" "${BATS_FORMATTER_TEST_DURATION}"
fi
}

pass_on_optional_data() {
if [[ "${BATS_FORMATTER_TEST_DURATION-x}" != x ]]; then
add_yaml_entry "duration_ms" "${BATS_FORMATTER_TEST_DURATION}"
fi
;;
'not ok '*)
if [[ "${BATS_FORMATTER_TEST_TIMEOUT-x}" != x ]]; then
add_yaml_entry "timeout_sec" "${BATS_FORMATTER_TEST_TIMEOUT}"
fi
}

bats_tap_stream_not_ok() { # <test index> <test name>
close_previous_yaml_block
number_of_printed_log_lines_for_this_test_so_far=0
timing_expr="(not ok [0-9]+ .+) in ([0-9])+ms$"
if [[ -n "${BATS_ENABLE_TIMING-}" ]]; then
if [[ "$line" =~ $timing_expr ]]; then
printf "%s\n" "${BASH_REMATCH[1]}"
add_yaml_entry "duration_ms" "${BASH_REMATCH[2]}"
else
echo "Could not match failure line to timing regex: $line" >&2
exit 1
fi
else
printf "%s\n" "${line}"
fi
;;
'# '*)

printf "not ok %d %s\n" "$1" "$2"
pass_on_optional_data
}

bats_tap_stream_skipped() { # <test index> <test name> <reason>
close_previous_yaml_block
number_of_printed_log_lines_for_this_test_so_far=0

printf "not ok %d %s # SKIP %s\n" "$1" "$2" "$3"
pass_on_optional_data
}

bats_tap_stream_comment() { # <comment text without leading '# '>
if [[ $number_of_printed_log_lines_for_this_test_so_far -eq 0 ]]; then
add_yaml_entry "message" "|" # use a multiline string for this entry
fi
((++number_of_printed_log_lines_for_this_test_so_far))
printf " %s\n" "${line:2}"
;;
'suite '*) ;;
esac
done
printf " %s\n" "$1"
}

bats_tap_stream_suite() { # <file name>
:
}

bats_tap_stream_unknown() { # <full line>
:
}

bats_parse_internal_extended_tap

# close the final block if there was one
close_previous_yaml_block
close_previous_yaml_block

0 comments on commit b1e7fed

Please sign in to comment.