Merge pull request #602 from martin-schulze-vireso/feature/issue_541_…

…external_formatters

Allow specifiyng external formatter via absolute path

docs/CHANGELOG.md

@@ -12,6 +12,9 @@ The format is based on [Keep a Changelog][kac] and this project adheres to
 
 ### Added
 
+* using external formatters via `--formatter <absolute path>` (also works for
+  `--report-formatter`) (#602)
+
 #### Documentation
 
 * update gotcha about negated statements: Recommend using `run !` on Bats

docs/source/usage.md

@@ -54,14 +54,33 @@ ok 1 addition using bc
 ok 2 addition using dc
 ```
 
-Test reports will be output in the executing directory, but may be placed elsewhere
-by specifying the `--output` flag.
+If you have your own formatter, you can use an absolute path to the executable
+to use it:
+
+```bash
+$ bats --formatter /absolute/path/to/my-formatter addition.bats
+addition using bc WORKED
+addition using dc FAILED
+```
+
+You can also generate test report files via `--report-formatter` which accepts
+the same options as `--formatter`. By default, the file is stored in the current
+workdir. However, it may be placed elsewhere by specifying the `--output` flag.
 
 ```text
-$ bats --formatter junit addition.bats --output /tmp
+$ bats --report-formatter junit addition.bats --output /tmp
 1..2
 ok 1 addition using bc
 ok 2 addition using dc
+
+$ cat /tmp/report.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites time="0.073">
+<testsuite name="addition.bats" tests="2" failures="0" errors="0" skipped="0">
+    <testcase classname="addition.bats" name="addition using bc" time="0.034" />
+    <testcase classname="addition.bats" name="addition using dc" time="0.039" />
+</testsuite>
+</testsuites>
 ```
 
 ## Parallel Execution

libexec/bats-core/bats

@@ -42,7 +42,7 @@ HELP_TEXT_HEADER
                             $BATS_END_CODE_QUOTE. Can also be set via $BATS_CODE_QUOTE_STYLE
   -f, --filter <regex>      Only run tests that match the regular expression
   -F, --formatter <type>    Switch between formatters: pretty (default),
-                              tap (default w/o term), tap13, junit
+                              tap (default w/o term), tap13, junit, /<absolute path to formatter>
   --gather-test-outputs-in <directory>
                             Gather the output of failing *and* passing tests
                             as files in directory (if existing, must be empty)
@@ -160,7 +160,7 @@ while [[ "$#" -ne 0 ]]; do
   -F | --formatter)
     shift
     # allow cat formatter to see extended output but don't advertise to users
-    if [[ $1 =~ ^(pretty|junit|tap|tap13|cat)$ ]]; then
+    if [[ $1 =~ ^(pretty|junit|tap|tap13|cat|/.*)$ ]]; then
       formatter="$1"
     else
       printf "Unknown formatter '%s', valid options are %s\n" "$1" "${VALID_FORMATTERS}"
@@ -317,6 +317,24 @@ if [[ "$formatter" == tap && -z "$report_formatter" ]]; then
   formatter="cat"
 fi
 
+bats_check_formatter() { # <formatter-path>
+  local -r formatter="$1"
+  if [[ ! -f "$formatter" ]]; then
+    printf "ERROR: Formatter '%s' is not readable!" "$formatter"
+    exit 1
+  elif [[ ! -x "$formatter" ]]; then
+    printf "ERROR: Formatter '%s' is not executable!" "$formatter"
+    exit 1
+  fi
+}
+
+if [[ $formatter == /* ]]; then # absolute paths are direct references to formatters
+  bats_check_formatter "$formatter"
+  interpolated_formatter="$formatter"
+else
+  interpolated_formatter="bats-format-${formatter}"
+fi
+
 if [[ "${#arguments[@]}" -eq 0 ]]; then
   abort 'Must specify at least one <test>'
 fi
@@ -326,17 +344,27 @@ if [[ -n "$report_formatter" ]]; then
   if [[ -z "$output" ]]; then
     output=.
   fi
-  case "$report_formatter" in
-    tap|tap13)
-      BATS_REPORT_FILE_NAME="report.tap"
-      ;;
-    junit)
-      BATS_REPORT_FILE_NAME="report.xml"
-      ;;
-    pretty)
-      BATS_REPORT_FILE_NAME="report.log"
-      ;;
-  esac
+  # only set BATS_REPORT_FILENAME if none was given
+  if [[ -z "${BATS_REPORT_FILENAME:-}" ]]; then
+    case "$report_formatter" in
+      tap|tap13)
+        BATS_REPORT_FILE_NAME="report.tap"
+        ;;
+      junit)
+        BATS_REPORT_FILE_NAME="report.xml"
+        ;;
+      *)
+        BATS_REPORT_FILE_NAME="report.log"
+        ;;
+    esac
+  fi
+fi
+
+if [[ $report_formatter == /* ]]; then # absolute paths are direct references to formatters
+  bats_check_formatter "$report_formatter"
+  interpolated_report_formatter="${report_formatter}"
+else
+  interpolated_report_formatter="bats-format-${report_formatter}"
 fi
 
 if [[ "${BATS_CODE_QUOTE_STYLE-BATS_CODE_QUOTE_STYLE_UNSET}" == BATS_CODE_QUOTE_STYLE_UNSET ]]; then
@@ -418,7 +446,12 @@ trap 'BATS_INTERRUPTED=true' INT # let the lower levels handle the interruption
 set -o pipefail execfail
 
 if [[ -n "$report_formatter" ]]; then
-  exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | tee >("bats-format-${report_formatter}" "${report_formatter_flags[@]}" >"${BATS_REPORT_OUTPUT_PATH}/${BATS_REPORT_FILE_NAME}") | bats_test_count_validator | "bats-format-${formatter}" "${formatter_flags[@]}"
+  exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | \
+    tee >("$interpolated_report_formatter" "${report_formatter_flags[@]}" >"${BATS_REPORT_OUTPUT_PATH}/${BATS_REPORT_FILE_NAME}") | \
+    bats_test_count_validator | \
+    "$interpolated_formatter" "${formatter_flags[@]}"
 else
-  exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | bats_test_count_validator | "bats-format-${formatter}" "${formatter_flags[@]}"
+  exec bats-exec-suite "${flags[@]}" "${filenames[@]}" | \
+    bats_test_count_validator | \
+    "$interpolated_formatter" "${formatter_flags[@]}"
 fi

libexec/bats-core/bats-exec-file

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -eET
 
-export flags=('--dummy-flag')
+flags=('--dummy-flag')
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 filter=''
 extended_syntax=''

man/bats.1.ronn

@@ -57,7 +57,8 @@ OPTIONS
   * `-f`, `--filter <regex>`:
     Filter test cases by names matching the regular expression
   * `-F`, `--formatter <type>`:
-    Switch between formatters: pretty (default), tap (default w/o term), tap13, junit
+    Switch between formatters: pretty (default), tap (default w/o term), tap13, junit,
+    `/<absolute path to formatter>`
   * `--gather-test-outputs-in <directory>`:
     Gather the output of failing *and* passing tests as files in directory
   * `-h`, `--help`:

test/bats.bats

@@ -63,15 +63,6 @@ setup() {
   [ "${lines[4]}" = "3 tests, 0 failures, 2 skipped" ]
 }
 
-@test "tap passing and skipping tests" {
-  run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_and_skipping.bats"
-  [ $status -eq 0 ]
-  [ "${lines[0]}" = "1..3" ]
-  [ "${lines[1]}" = "ok 1 a passing test" ]
-  [ "${lines[2]}" = "ok 2 a skipped test with no reason # skip" ]
-  [ "${lines[3]}" = "ok 3 a skipped test with a reason # skip for a really good reason" ]
-}
-
 @test "summary passing and failing tests" {
   run filter_control_sequences bats -p "$FIXTURE_ROOT/failing_and_passing.bats"
   [ $status -eq 0 ]
@@ -84,15 +75,6 @@ setup() {
   [ "${lines[6]}" = "3 tests, 1 failure, 1 skipped" ]
 }
 
-@test "tap passing, failing and skipping tests" {
-  run filter_control_sequences bats --formatter tap "$FIXTURE_ROOT/passing_failing_and_skipping.bats"
-  [ $status -eq 0 ]
-  [ "${lines[0]}" = "1..3" ]
-  [ "${lines[1]}" = "ok 1 a passing test" ]
-  [ "${lines[2]}" = "ok 2 a skipping test # skip" ]
-  [ "${lines[3]}" = "not ok 3 a failing test" ]
-}
-
 @test "BATS_CWD is correctly set to PWD as validated by bats_trim_filename" {
   local trimmed
   bats_trim_filename "$PWD/foo/bar" 'trimmed'
@@ -266,16 +248,6 @@ setup() {
   [ "${lines[2]}" = "ok 2 a skipped test with a reason # skip a reason" ]
 }
 
-@test "skipped test with parens (pretty formatter)" {
-  run bats --pretty "$FIXTURE_ROOT/skipped_with_parens.bats"
-  [ $status -eq 0 ]
-
-  # Some systems (Alpine, for example) seem to emit an extra whitespace into
-  # entries in the 'lines' array when a carriage return is present from the
-  # pretty formatter.  This is why a '+' is used after the 'skipped' note.
-  [[ "${lines[*]}" =~ "- a skipped test with parentheses in the reason (skipped: "+"a reason (with parentheses))" ]]
-}
-
 @test "extended syntax" {
   emulate_bats_env
   run bats-exec-suite -x "$FIXTURE_ROOT/failing_and_passing.bats"
@@ -320,25 +292,6 @@ setup() {
   [[ "${lines[3]}" =~ $regex ]]
 }
 
-@test "pretty and tap formats" {
-  run bats --formatter tap "$FIXTURE_ROOT/passing.bats"
-  tap_output="$output"
-  [ $status -eq 0 ]
-
-  run bats --pretty "$FIXTURE_ROOT/passing.bats"
-  pretty_output="$output"
-  [ $status -eq 0 ]
-
-  [ "$tap_output" != "$pretty_output" ]
-}
-
-@test "pretty formatter bails on invalid tap" {
-  run bats-format-pretty < <(printf "This isn't TAP!\nGood day to you\n")
-  [ $status -eq 0 ]
-  [ "${lines[0]}" = "This isn't TAP!" ]
-  [ "${lines[1]}" = "Good day to you" ]
-}
-
 @test "single-line tests" {
   run bats "$FIXTURE_ROOT/single_line_no_shellcheck.bats"
   [ $status -eq 1 ]
@@ -721,34 +674,6 @@ END_OF_ERR_MSG
   [ "$(find "$TEST_TMPDIR" -name '*.src' | wc -l)" -eq 1 ]
 }
 
-@test "All formatters (except cat) implement the callback interface" {
-  cd "$BATS_ROOT/libexec/bats-core/"
-  for formatter in bats-format-*; do
-    # the cat formatter is not expected to implement this interface
-    if [[ "$formatter" == *"bats-format-cat" ]]; then
-      continue
-    fi
-    tested_at_least_one_formatter=1
-    echo "Formatter: ${formatter}"
-    # the replay should be possible without errors
-    bash -u "$formatter" >/dev/null <<EOF
-1..1
-suite "$BATS_FIXTURE_ROOT/failing.bats"
-# output from setup_file
-begin 1 test_a_failing_test
-# fd3 output from test
-not ok 1 a failing test
-# (in test file test/fixtures/bats/failing.bats, line 4)
-#   \`eval "( exit ${STATUS:-1} )"' failed
-begin 2 test_a_successful_test
-ok 2 a succesful test
-unknown line
-EOF
-  done
-
-  [[ -n "$tested_at_least_one_formatter" ]]
-}
-
 @test "run should exit if tmpdir exist" {
   local dir
   dir=$(mktemp -d "${BATS_RUN_TMPDIR}/BATS_RUN_TMPDIR_TEST.XXXXXX")
@@ -1061,24 +986,6 @@ EOF
   [[ "$BATS_RUN_COMMAND" == "bats BATS_RUN_COMMAND: test content of variable" ]]
 }
 
-@test "pretty formatter summary is colorized red on failure" {
-  bats_require_minimum_version 1.5.0
-  run -1 bats --pretty "$FIXTURE_ROOT/failing.bats"
-
-  [ "${lines[4]}" == $'\033[0m\033[31;1m' ] # TODO: avoid checking for the leading reset too
-  [ "${lines[5]}" == '1 test, 1 failure' ]
-  [ "${lines[6]}" == $'\033[0m' ]
-}
-
-@test "pretty formatter summary is colorized green on success" {
-  bats_require_minimum_version 1.5.0
-  run -0 bats --pretty "$FIXTURE_ROOT/passing.bats"
-
-  [ "${lines[2]}" == $'\033[0m\033[32;1m' ] # TODO: avoid checking for the leading reset too
-  [ "${lines[3]}" == '1 test, 0 failures' ]
-  [ "${lines[4]}" == $'\033[0m' ]
-}
-
 @test "--print-output-on-failure works as expected" {
   run bats --print-output-on-failure --show-output-of-passing-tests "$FIXTURE_ROOT/print_output_on_failure.bats"
   [ "${lines[0]}" == '1..3' ]

test/fixtures/formatter/dummy-formatter

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cat >/dev/null # ignore input 
+
+echo "Dummy Formatter!"

test/fixtures/formatter/failing.bats

@@ -0,0 +1,5 @@
+@test "a failing test" {
+  true
+  true
+  eval "( exit ${STATUS:-1} )"
+}

test/fixtures/formatter/passing.bats

@@ -0,0 +1,3 @@
+@test "a passing test" {
+  true
+}

test/fixtures/formatter/passing_and_skipping.bats

@@ -0,0 +1,11 @@
+@test "a passing test" {
+  true
+}
+
+@test "a skipped test with no reason" {
+  skip
+}
+
+@test "a skipped test with a reason" {
+  skip "for a really good reason"
+}

test/fixtures/formatter/passing_failing_and_skipping.bats

@@ -0,0 +1,11 @@
+@test "a passing test" {
+  true
+}
+
+@test "a skipping test" {
+  skip
+}
+
+@test "a failing test" {
+  false
+}

test/fixtures/formatter/skipped_with_parens.bats

@@ -0,0 +1,3 @@
+@test "a skipped test with parentheses in the reason" {
+  skip "a reason (with parentheses)"
+}