diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 733499cb2b..d8c22677aa 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,11 @@ The format is based on [Keep a Changelog][kac] and this project adheres to * `colon`: `file.bats:1` * `uri`: `file:///path/to/file.bats:1` * `custom`: define your own formatter in `bats_format_file_line_reference_custom` +* add `bats:focus` tag to run only focused tests (#679) + +### Documentation + +* add `--help` text and `man` page content for `--filter-tags` (#679) ### Fixed diff --git a/docs/source/writing-tests.md b/docs/source/writing-tests.md index bbe1a9cb22..004409a1fa 100644 --- a/docs/source/writing-tests.md +++ b/docs/source/writing-tests.md @@ -70,9 +70,20 @@ They must not contain empty tags like `test_tags=,b` (first tag is empty), `test_tags=a,,c`, `test_tags=a, ,c` (second tag is only whitespace/empty), `test_tags=a,b,` (third tag is empty). -Every tag starting with a `bats:` (case insensitive!) is reserved for Bats' +Every tag starting with `bats:` (case insensitive!) is reserved for Bats' internal use. +### Special tags + +#### Focusing on tests with `bats:focus` tag + +If a test with the tag `bats:focus` is encountered in a test suite, +all other tests will be filtered out and only those tagged with this tag will be executed. + +In focus mode, the exit code of successful runs will be overriden to 1 to prevent CI from silently running on a subset of tests due to an accidentally commited `bats:focus` tag. +Should you require the true exit code, e.g. for a `git bisect` operation, you can disable this behavior by setting +`BATS_NO_FAIL_FOCUS_RUN=1` when running `bats`, but make sure not to commit this to CI! + ### Filtering execution Tags can be used for more finegrained filtering of which tests to run via `--filter-tags`. @@ -91,6 +102,8 @@ This means multiple `--filter-tags` form a boolean disjunction. A query of `--filter-tags a,!b --filter-tags b,c` can be translated to: Execute only tests that (have tag a, but not tag b) or (have tag b and c). +An empty tag list matches tests without tags. + ## Comment syntax External tools (like `shellcheck`, `shfmt`, and various IDE's) may not support diff --git a/libexec/bats-core/bats b/libexec/bats-core/bats index 726975608e..bfd5975246 100755 --- a/libexec/bats-core/bats +++ b/libexec/bats-core/bats @@ -51,6 +51,11 @@ HELP_TEXT_HEADER Valid values are: failed - runs tests that failed or were not present in the last run missed - runs tests that were not present in the last run + --filter-tags + Only run tests that match all the tags in the list (&&). + You can negate a tag via prepending '!'. + Specifying this flag multiple times allows for logical or (||): + `--filter-tags A,B --filter-tags A,!C` matches tags (A && B) || (A && !C) -F, --formatter Switch between formatters: pretty (default), tap (default w/o term), tap13, junit, / --gather-test-outputs-in diff --git a/libexec/bats-core/bats-exec-suite b/libexec/bats-core/bats-exec-suite index cc8d429aa9..ecf189b240 100755 --- a/libexec/bats-core/bats-exec-suite +++ b/libexec/bats-core/bats-exec-suite @@ -108,7 +108,7 @@ fi # create a file that contains all (filtered) tests to run from all files TESTS_LIST_FILE="${BATS_RUN_TMPDIR}/test_list_file.txt" - +focus_mode= bats_gather_tests() { local line test_line tags all_tests=() @@ -132,6 +132,23 @@ bats_gather_tests() { else tags=() fi + # is this test focused? + if bats_all_in tags 'bats:focus'; then + if [[ $focus_mode == 1 ]]; then + # focused tests in focus mode should just be registered + : + else + # the current test enables focus mode ... + focus_mode=1 + # ... -> remove previously found, unfocused tests + all_tests=() + : > "$TESTS_LIST_FILE" + fi + elif [[ $focus_mode == 1 ]]; then + # the current test is not focused but focus mode is enabled -> filter out + continue + # no else -> unfocused tests outside focus mode should just be registered + fi if [[ ${#filter_tags_list[@]} -gt 0 ]]; then local match= for filter_tags in "${filter_tags_list[@]}"; do @@ -300,6 +317,10 @@ fi # only abort on the lowest levels trap 'BATS_INTERRUPTED=true' INT +if [[ -n "$focus_mode" ]]; then + printf "WARNING: This test run only contains tests tagged \`bats:focus\`!\n" +fi + bats_exec_suite_status=0 printf '1..%d\n' "${test_count}" @@ -431,4 +452,13 @@ fi set -eET bats_run_teardown_suite -exit "$bats_exec_suite_status" +if [[ "$focus_mode" == 1 && $bats_exec_suite_status -eq 0 ]]; then + if [[ ${BATS_NO_FAIL_FOCUS_RUN-} == 1 ]]; then + printf "WARNING: This test run only contains tests tagged \`bats:focus\`!\n" + else + printf "Marking test run as failed due to \`bats:focus\` tag. (Set \`BATS_NO_FAIL_FOCUS_RUN=1\` to disable.)\n" >&2 + bats_exec_suite_status=1 + fi +fi + +exit "$bats_exec_suite_status" # the actual exit code will be set by the exit trap using bats_exec_suite_status diff --git a/man/bats.1 b/man/bats.1 index fcf5b92e53..c806f575f5 100644 --- a/man/bats.1 +++ b/man/bats.1 @@ -1,149 +1,143 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BATS" "1" "November 2021" "bats-core" "Bash Automated Testing System" -. +.\" generated with Ronn-NG/v0.9.1 +.\" http://github.com/apjanke/ronn-ng/tree/0.9.1 +.TH "BATS" "1" "November 2022" "bats-core" "Bash Automated Testing System" .SH "NAME" \fBbats\fR \- Bash Automated Testing System -. .SH "SYNOPSIS" Usage: bats [OPTIONS] \fItests\fR bats [\-h | \-v] -. .P \fItests\fR is the path to a Bats test file, or the path to a directory containing Bats test files (ending with "\.bats") -. .SH "DESCRIPTION" Bats is a TAP\-compliant testing framework for Bash\. It provides a simple way to verify that the UNIX programs you write behave as expected\. -. .P A Bats test file is a Bash script with special syntax for defining test cases\. Under the hood, each test case is just a function with a description\. -. .P Test cases consist of standard shell commands\. Bats makes use of Bash\'s \fBerrexit\fR (\fBset \-e\fR) option when running test cases\. If every command in the test case exits with a \fB0\fR status code (success), the test passes\. In this way, each line is an assertion of truth\. -. .P See \fBbats\fR(7) for more information on writing Bats tests\. -. .SH "RUNNING TESTS" To run your tests, invoke the \fBbats\fR interpreter with a path to a test file\. The file\'s test cases are run sequentially and in isolation\. If all the test cases pass, \fBbats\fR exits with a \fB0\fR status code\. If there are any failures, \fBbats\fR exits with a \fB1\fR status code\. -. .P You can invoke the \fBbats\fR interpreter with multiple test file arguments, or with a path to a directory containing multiple \fB\.bats\fR files\. Bats will run each test file individually and aggregate the results\. If any test case fails, \fBbats\fR exits with a \fB1\fR status code\. -. -.SH "OPTIONS" -. -.IP "\(bu" 4 -\fB\-c\fR, \fB\-\-count\fR: Count the number of test cases without running any tests -. -.IP "\(bu" 4 -\fB\-\-code\-quote\-style