From a84b7e15044ace0c0f744eb25c7fd36d6ff33f6b Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Sat, 12 Nov 2022 00:03:45 +0100 Subject: [PATCH 1/7] Add special tag bats:focus --- libexec/bats-core/bats-exec-suite | 19 ++++++++++++++++++- test/bats.bats | 8 ++++++++ test/fixtures/bats/focus.bats | 8 ++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/bats/focus.bats diff --git a/libexec/bats-core/bats-exec-suite b/libexec/bats-core/bats-exec-suite index cc8d429aa9..ad88149d60 100755 --- a/libexec/bats-core/bats-exec-suite +++ b/libexec/bats-core/bats-exec-suite @@ -108,7 +108,7 @@ fi # create a file that contains all (filtered) tests to run from all files TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt" - +focus_mode= bats_gather_tests() { local line test_line tags all_tests=() @@ -132,6 +132,23 @@ bats_gather_tests() { else tags=() fi + # is this test focused? + if bats_all_in tags 'bats:focus'; then + if [[ $focus_mode == 1 ]]; then + # focused tests in focus mode should just be registered + : + else + # the current test enables focus mode ... + focus_mode=1 + # ... -> remove previously found, unfocused tests + all_tests=() + : > "$TESTS_LIST_FILE" + fi + elif [[ $focus_mode == 1 ]]; then + # the current test is not focused but focus mode is enabled -> filter out + continue + # no else -> unfocused tests outside focus mode should just be registered + fi if [[ ${#filter_tags_list[@]} -gt 0 ]]; then local match= for filter_tags in "${filter_tags_list[@]}"; do diff --git a/test/bats.bats b/test/bats.bats index 63b72ad3d6..f5698cf3d5 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -1521,3 +1521,11 @@ enforce_own_process_group() { reentrant_run -1 bats --line-reference-format custom "$FIXTURE_ROOT/failing.bats" [ "${lines[2]}" == "# (in test file $RELATIVE_FIXTURE_ROOT/failing.bats<-4)" ] } + +@test "Focused tests filter out other tests" { + bats_require_minimum_version 1.5.0 + reentrant_run -0 bats "$FIXTURE_ROOT/focus.bats" + [ "${lines[0]}" == '1..1' ] + [ "${lines[1]}" == 'ok 1 focused' ] + [ "${#lines[@]}" == 2 ] +} diff --git a/test/fixtures/bats/focus.bats b/test/fixtures/bats/focus.bats new file mode 100644 index 0000000000..d9c23f0cd7 --- /dev/null +++ b/test/fixtures/bats/focus.bats @@ -0,0 +1,8 @@ +@test "unfocused" { + false +} + +# bats test_tags=bats:focus +@test "focused" { + true +} \ No newline at end of file From 6eb1f1f726f8008faf927208a658990d662427f4 Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Tue, 22 Nov 2022 23:56:44 +0100 Subject: [PATCH 2/7] Override exit code in focus mode unless disabled --- libexec/bats-core/bats-exec-suite | 15 ++++++++++++++- test/bats.bats | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/libexec/bats-core/bats-exec-suite b/libexec/bats-core/bats-exec-suite index ad88149d60..ecf189b240 100755 --- a/libexec/bats-core/bats-exec-suite +++ b/libexec/bats-core/bats-exec-suite @@ -317,6 +317,10 @@ fi # only abort on the lowest levels trap 'BATS_INTERRUPTED=true' INT +if [[ -n "$focus_mode" ]]; then + printf "WARNING: This test run only contains tests tagged \`bats:focus\`!\n" +fi + bats_exec_suite_status=0 printf '1..%d\n' "${test_count}" @@ -448,4 +452,13 @@ fi set -eET bats_run_teardown_suite -exit "$bats_exec_suite_status" +if [[ "$focus_mode" == 1 && $bats_exec_suite_status -eq 0 ]]; then + if [[ ${BATS_NO_FAIL_FOCUS_RUN-} == 1 ]]; then + printf "WARNING: This test run only contains tests tagged \`bats:focus\`!\n" + else + printf "Marking test run as failed due to \`bats:focus\` tag. (Set \`BATS_NO_FAIL_FOCUS_RUN=1\` to disable.)\n" >&2 + bats_exec_suite_status=1 + fi +fi + +exit "$bats_exec_suite_status" # the actual exit code will be set by the exit trap using bats_exec_suite_status diff --git a/test/bats.bats b/test/bats.bats index f5698cf3d5..4107639d68 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -1529,3 +1529,25 @@ enforce_own_process_group() { [ "${lines[1]}" == 'ok 1 focused' ] [ "${#lines[@]}" == 2 ] } + +@test "Focus tests filter out other tests and override exit code" { + bats_require_minimum_version 1.5.0 + # expect exit 1: focus mode always fails tests + reentrant_run -1 bats "$FIXTURE_ROOT/focus.bats" + [ "${lines[0]}" == 'WARNING: This test run only contains tests tagged `bats:focus`!' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'ok 1 focused' ] + [ "${lines[3]}" == 'Marking test run as failed due to `bats:focus` tag. (Set `BATS_NO_FAIL_FOCUS_RUN=1` to disable.)' ] + [ "${#lines[@]}" == 4 ] +} + +@test "Focus tests with BATS_NO_FAIL_FOCUS_RUN=1 does not override exit code" { + bats_require_minimum_version 1.5.0 + REENTRANT_RUN_PRESERVE+=(BATS_NO_FAIL_FOCUS_RUN) + BATS_NO_FAIL_FOCUS_RUN=1 reentrant_run -0 bats "$FIXTURE_ROOT/focus.bats" + [ "${lines[0]}" == 'WARNING: This test run only contains tests tagged `bats:focus`!' ] + [ "${lines[1]}" == '1..1' ] + [ "${lines[2]}" == 'ok 1 focused' ] + [ "${lines[3]}" == 'WARNING: This test run only contains tests tagged `bats:focus`!' ] + [ "${#lines[@]}" == 4 ] +} From b793674d79f833b709a26837965755fac42457c7 Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Wed, 23 Nov 2022 00:50:47 +0100 Subject: [PATCH 3/7] Add online docs for bats:focus --- docs/source/writing-tests.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/writing-tests.md b/docs/source/writing-tests.md index bbe1a9cb22..004409a1fa 100644 --- a/docs/source/writing-tests.md +++ b/docs/source/writing-tests.md @@ -70,9 +70,20 @@ They must not contain empty tags like `test_tags=,b` (first tag is empty), `test_tags=a,,c`, `test_tags=a, ,c` (second tag is only whitespace/empty), `test_tags=a,b,` (third tag is empty). -Every tag starting with a `bats:` (case insensitive!) is reserved for Bats' +Every tag starting with `bats:` (case insensitive!) is reserved for Bats' internal use. +### Special tags + +#### Focusing on tests with `bats:focus` tag + +If a test with the tag `bats:focus` is encountered in a test suite, +all other tests will be filtered out and only those tagged with this tag will be executed. + +In focus mode, the exit code of successful runs will be overriden to 1 to prevent CI from silently running on a subset of tests due to an accidentally commited `bats:focus` tag. +Should you require the true exit code, e.g. for a `git bisect` operation, you can disable this behavior by setting +`BATS_NO_FAIL_FOCUS_RUN=1` when running `bats`, but make sure not to commit this to CI! + ### Filtering execution Tags can be used for more finegrained filtering of which tests to run via `--filter-tags`. @@ -91,6 +102,8 @@ This means multiple `--filter-tags` form a boolean disjunction. A query of `--filter-tags a,!b --filter-tags b,c` can be translated to: Execute only tests that (have tag a, but not tag b) or (have tag b and c). +An empty tag list matches tests without tags. + ## Comment syntax External tools (like `shellcheck`, `shfmt`, and various IDE's) may not support From c41c518dc252cb8f827b56b21d5e3f8132a2f96e Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Wed, 23 Nov 2022 00:51:08 +0100 Subject: [PATCH 4/7] Add offline docs for tagging and bats:focus fixes (#675) --- libexec/bats-core/bats | 5 + man/bats.1 | 190 +++++++++++++++++++------------------- man/bats.1.ronn | 44 ++++++++- man/bats.7 | 204 ++++++++++++++--------------------------- man/bats.7.ronn | 63 +++++++++++++ 5 files changed, 270 insertions(+), 236 deletions(-) diff --git a/libexec/bats-core/bats b/libexec/bats-core/bats index 726975608e..bfd5975246 100755 --- a/libexec/bats-core/bats +++ b/libexec/bats-core/bats @@ -51,6 +51,11 @@ HELP_TEXT_HEADER Valid values are: failed - runs tests that failed or were not present in the last run missed - runs tests that were not present in the last run + --filter-tags + Only run tests that match all the tags in the list (&&). + You can negate a tag via prepending '!'. + Specifying this flag multiple times allows for logical or (||): + `--filter-tags A,B --filter-tags A,!C` matches tags (A && B) || (A && !C) -F, --formatter Switch between formatters: pretty (default), tap (default w/o term), tap13, junit, / --gather-test-outputs-in diff --git a/man/bats.1 b/man/bats.1 index fcf5b92e53..c806f575f5 100644 --- a/man/bats.1 +++ b/man/bats.1 @@ -1,149 +1,143 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BATS" "1" "November 2021" "bats-core" "Bash Automated Testing System" -. +.\" generated with Ronn-NG/v0.9.1 +.\" http://github.com/apjanke/ronn-ng/tree/0.9.1 +.TH "BATS" "1" "November 2022" "bats-core" "Bash Automated Testing System" .SH "NAME" \fBbats\fR \- Bash Automated Testing System -. .SH "SYNOPSIS" Usage: bats [OPTIONS] \fItests\fR bats [\-h | \-v] -. .P \fItests\fR is the path to a Bats test file, or the path to a directory containing Bats test files (ending with "\.bats") -. .SH "DESCRIPTION" Bats is a TAP\-compliant testing framework for Bash\. It provides a simple way to verify that the UNIX programs you write behave as expected\. -. .P A Bats test file is a Bash script with special syntax for defining test cases\. Under the hood, each test case is just a function with a description\. -. .P Test cases consist of standard shell commands\. Bats makes use of Bash\'s \fBerrexit\fR (\fBset \-e\fR) option when running test cases\. If every command in the test case exits with a \fB0\fR status code (success), the test passes\. In this way, each line is an assertion of truth\. -. .P See \fBbats\fR(7) for more information on writing Bats tests\. -. .SH "RUNNING TESTS" To run your tests, invoke the \fBbats\fR interpreter with a path to a test file\. The file\'s test cases are run sequentially and in isolation\. If all the test cases pass, \fBbats\fR exits with a \fB0\fR status code\. If there are any failures, \fBbats\fR exits with a \fB1\fR status code\. -. .P You can invoke the \fBbats\fR interpreter with multiple test file arguments, or with a path to a directory containing multiple \fB\.bats\fR files\. Bats will run each test file individually and aggregate the results\. If any test case fails, \fBbats\fR exits with a \fB1\fR status code\. -. -.SH "OPTIONS" -. -.IP "\(bu" 4 -\fB\-c\fR, \fB\-\-count\fR: Count the number of test cases without running any tests -. -.IP "\(bu" 4 -\fB\-\-code\-quote\-style