diff --git a/README.md b/README.md index 235bf1ee..2e0dd5cc 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,39 @@ You can force TAP output from a terminal by invoking Bats with the ok 1 addition using bc ok 2 addition using dc +As stated previously you can redirect the output to a file and it +will be in TAP format, but you can specify to write the output to +a file using the '-o' option which will be in pretty format. + + $ bats -o addition.tap addition.bats + $ cat addition.tap + + ✓ addition using bc + ✓ addition using dc + + 2 tests, 0 failures + +You may also tell it to do the same in TAP format using --tap parameter. +Finally, if output has been redirected using -o then you may also tell +bats to release output from tests and commands run in tests on STDOUT. +For instance, if you have a test file like: + +```bats +@test "echo something" { + echo something + [ 0 -eq 0 ] +} +``` + +Then you would get this: + + $ bats --tap -r -o echo.tap echo.bats + something + $ cat echo.tap + 1..1 + ok 1 echo something + + ### Test suites You can invoke the `bats` interpreter with multiple test file diff --git a/libexec/bats b/libexec/bats index 71f392f7..e77e2b8e 100755 --- a/libexec/bats +++ b/libexec/bats @@ -1,13 +1,14 @@ #!/usr/bin/env bash set -e + version() { echo "Bats 0.4.0" } usage() { version - echo "Usage: bats [-c] [-p | -t] [ ...]" + echo "Usage: bats [-c] [-p | -t] [-r] [-o ] [ ...]" } help() { @@ -20,6 +21,8 @@ help() { echo " -h, --help Display this help message" echo " -p, --pretty Show results in pretty format (default for terminals)" echo " -t, --tap Show results in TAP format" + echo " -o Write test results to " + echo " -r Release output from tests to STDOUT" echo " -v, --version Display the version number" echo echo " For more information, see https://github.com/sstephenson/bats" @@ -52,65 +55,91 @@ expand_path() { } || echo "$1" } -BATS_LIBEXEC="$(abs_dirname "$0")" +export BATS_LIBEXEC="$(abs_dirname "$0")" export BATS_PREFIX="$(abs_dirname "$BATS_LIBEXEC")" export BATS_CWD="$(abs_dirname .)" export PATH="$BATS_LIBEXEC:$PATH" -options=() -arguments=() -for arg in "$@"; do - if [ "${arg:0:1}" = "-" ]; then - if [ "${arg:1:1}" = "-" ]; then - options[${#options[*]}]="${arg:2}" - else - index=1 - while option="${arg:$index:1}"; do - [ -n "$option" ] || break - options[${#options[*]}]="$option" - let index+=1 - done - fi - else - arguments[${#arguments[*]}]="$arg" - fi -done +source $BATS_LIBEXEC/common_functions.shrc -unset count_flag pretty +unset pass_the_opts pretty [ -t 0 ] && [ -t 1 ] && pretty="1" [ -n "$CI" ] && pretty="" -for option in "${options[@]}"; do - case "$option" in - "h" | "help" ) - help - exit 0 - ;; - "v" | "version" ) - version - exit 0 - ;; - "c" | "count" ) - count_flag="-c" - ;; - "t" | "tap" ) - pretty="" - ;; - "p" | "pretty" ) - pretty="1" - ;; - * ) - usage >&2 - exit 1 - ;; +while getopts ":hvctpro:-:" opt; do + if [[ $opt == '-' ]]; then + case $OPTARG in + 'pretty') + opt=p;; + 'tap') + opt=t;; + 'help') + opt=h;; + 'version') + opt=v;; + *) + if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then + echo "Unknown option --${OPTARG}" >&2 + fi + exit 1 + ;; + esac + fi + case $opt in + h) + help + exit 0 + ;; + v) + version + exit 0 + ;; + c) + pass_the_opts="$pass_the_opts -c" + ;; + r) + pass_the_opts="$pass_the_opts -r" + BATS_RELEASE_OUTPUT=1 + ;; + t) + pretty="" + ;; + p) + pretty="1" + ;; + o) + TMPFILE1=$(createTempFile) + pass_the_opts="$pass_the_opts -o $TMPFILE1" + BATS_TAPOUT_FILE=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; esac done +shift $((OPTIND-1)) + +arguments=("${@}") if [ "${#arguments[@]}" -eq 0 ]; then usage >&2 exit 1 fi +bats_exit_trap() { + # remove rubbish + if [[ -e $TMPFILE1 ]]; then + rm -f $TMPFILE1 + fi +} +trap "bats_exit_trap" exit + + filenames=() for filename in "${arguments[@]}"; do if [ -d "$filename" ]; then @@ -139,4 +168,22 @@ else fi set -o pipefail execfail -exec "$command" $count_flag $extended_syntax_flag "${filenames[@]}" | "$formatter" + +if [[ -n $BATS_RELEASE_OUTPUT && -z $BATS_TAPOUT_FILE ]]; then + # covers: -r + echo "ERROR: Cannot release output of tests to STDOUT and produce TAP output to STDOUT" 2>&1 + exit 1 +else + if [[ -n $BATS_TAPOUT_FILE ]]; then + # covers: -r -o t.tap || -o t.tap + set +e + "$command" $pass_the_opts $extended_syntax_flag "${filenames[@]}" + status=$? + set -e + cat $TMPFILE1 | "$formatter" > $BATS_TAPOUT_FILE + exit $status + else + # (near) original command: no tapout file, no release to STDOUT - will spew test results on STDOUT + exec "$command" $pass_the_opts $extended_syntax_flag "${filenames[@]}" | "$formatter" + fi +fi diff --git a/libexec/bats-exec-suite b/libexec/bats-exec-suite index 29ab255d..ad2cdb0d 100755 --- a/libexec/bats-exec-suite +++ b/libexec/bats-exec-suite @@ -1,17 +1,38 @@ #!/usr/bin/env bash set -e -count_only_flag="" -if [ "$1" = "-c" ]; then - count_only_flag=1 - shift -fi +source $BATS_LIBEXEC/common_functions.shrc +unset pass_the_opts +count_only_flag="" extended_syntax_flag="" -if [ "$1" = "-x" ]; then - extended_syntax_flag="-x" - shift -fi +while getopts ":o:cxr" opt; do + case $opt in + c) + count_only_flag=1 + ;; + x) + extended_syntax_flag="-x" + pass_the_opts="$pass_the_opts -x" + ;; + o) + export BATS_OUTPUT_FILE=$OPTARG + pass_the_opts="$pass_the_opts -o $BATS_OUTPUT_FILE" + ;; + r) + pass_the_opts="$pass_the_opts -r" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done +shift $((OPTIND-1)) trap "kill 0; exit 1" int @@ -20,36 +41,61 @@ for filename in "$@"; do let count+="$(bats-exec-test -c "$filename")" done +# Set up the location of TAP output and location of run outputs +if [[ -n $BATS_OUTPUT_FILE ]]; then + # Store output in temp file and overwrite BATS_OUTPUT_FILE with it later + TMP_TAP_FILE=$(createTempFile) + exec 4>$TMP_TAP_FILE +else + exec 4<&1 # copy STDOUT file descriptor and use that instead of assuming STDOUT as the file descriptor of outputs +fi + if [ -n "$count_only_flag" ]; then - echo "$count" + >&4 echo "$count" exit fi -echo "1..$count" +>&4 echo "1..$count" status=0 offset=0 for filename in "$@"; do index=0 + if [[ -n $BATS_OUTPUT_FILE ]]; then + echo -n "" > $BATS_OUTPUT_FILE # zero out the file +set +e + bats-exec-test $pass_the_opts "$filename" +set -e + exec 5<$BATS_OUTPUT_FILE + else + exec 5< <( bats-exec-test $pass_the_opts "$filename" ) + fi + { - IFS= read -r # 1..n + IFS= read -r myline # 1..n while IFS= read -r line; do case "$line" in "begin "* ) let index+=1 - echo "${line/ $index / $(($offset + $index)) }" + >&4 echo "${line/ $index / $(($offset + $index)) }" ;; "ok "* | "not ok "* ) [ -n "$extended_syntax_flag" ] || let index+=1 - echo "${line/ $index / $(($offset + $index)) }" + >&4 echo "${line/ $index / $(($offset + $index)) }" [ "${line:0:6}" != "not ok" ] || status=1 ;; * ) - echo "$line" + >&4 echo "$line" ;; esac done - } < <( bats-exec-test $extended_syntax_flag "$filename" ) + } <&5 + offset=$(($offset + $index)) done +if [[ -n $BATS_OUTPUT_FILE ]]; then + # move temp file over BATS_OUTPUT_FILE + mv -f $TMP_TAP_FILE $BATS_OUTPUT_FILE +fi + exit "$status" diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 8f3bd510..03dadc73 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -3,17 +3,35 @@ set -e set -E set -T -BATS_COUNT_ONLY="" -if [ "$1" = "-c" ]; then - BATS_COUNT_ONLY=1 - shift -fi +source common_functions.shrc +BATS_COUNT_ONLY="" BATS_EXTENDED_SYNTAX="" -if [ "$1" = "-x" ]; then - BATS_EXTENDED_SYNTAX="$1" - shift -fi +while getopts ":o:cxr" opt; do + case $opt in + c) + BATS_COUNT_ONLY=1 + ;; + x) + BATS_EXTENDED_SYNTAX="-x" + ;; + o) + export BATS_OUTPUT_FILE=$OPTARG + ;; + r) + export BATS_RELEASE_TESTOUTPUT=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done +shift $((OPTIND-1)) BATS_TEST_FILENAME="$1" if [ -z "$BATS_TEST_FILENAME" ]; then @@ -55,14 +73,23 @@ run() { set +e set +E set +T - output="$("$@" 2>&1)" - status="$?" + if [[ -n $BATS_RELEASE_TESTOUTPUT ]]; then + local TMPFILE=$(createTempFile) + "$@" > >(tee $TMPFILE) 2>&1 + status="$?" + output=$(cat $TMPFILE) + rm -f $TMPFILE + else + output="$("$@" 2>&1)" + status="$?" + fi oldIFS=$IFS IFS=$'\n' lines=($output) + IFS=$oldIFS + [ -z "$e" ] || set -e [ -z "$E" ] || set -E [ -z "$T" ] || set -T - IFS=$oldIFS } setup() { @@ -113,7 +140,6 @@ bats_capture_stack_trace() { let index+=1 fi done - BATS_SOURCE="$(bats_frame_filename "${BATS_CURRENT_STACK_TRACE[0]}")" BATS_LINENO="$(bats_frame_lineno "${BATS_CURRENT_STACK_TRACE[0]}")" } @@ -244,11 +270,12 @@ bats_exit_trap() { local skipped trap - err exit + skipped="" if [ -n "$BATS_TEST_SKIPPED" ]; then skipped=" # skip" if [ "1" != "$BATS_TEST_SKIPPED" ]; then - skipped+=" ($BATS_TEST_SKIPPED)" + skipped+=" $BATS_TEST_SKIPPED" fi fi @@ -259,7 +286,7 @@ bats_exit_trap() { sed -e "s/^/# /" < "$BATS_OUT" >&3 status=1 else - echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 + echo "ok ${BATS_TEST_NUMBER} ${BATS_TEST_DESCRIPTION}${skipped}" >&3 status=0 fi @@ -268,7 +295,7 @@ bats_exit_trap() { } bats_perform_tests() { - echo "1..$#" + echo "1..$#" >&3 test_number=1 status=0 for test_name in "$@"; do @@ -283,7 +310,7 @@ bats_perform_test() { if [ "$(type -t "$BATS_TEST_NAME" || true)" = "function" ]; then BATS_TEST_NUMBER="$2" if [ -z "$BATS_TEST_NUMBER" ]; then - echo "1..1" + echo "1..1" >&3 BATS_TEST_NUMBER="1" fi @@ -292,23 +319,22 @@ bats_perform_test() { trap "bats_debug_trap \"\$BASH_SOURCE\"" debug trap "bats_error_trap" err trap "bats_teardown_trap" exit - "$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1 - BATS_TEST_COMPLETED=1 + if [[ -n $BATS_RELEASE_TESTOUTPUT ]]; then + 2>&1 "$BATS_TEST_NAME" > >(tee "$BATS_OUT") + else + "$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1 + fi + BATS_TEST_COMPLETED=1 else echo "bats: unknown test name \`$BATS_TEST_NAME'" >&2 exit 1 fi } -if [ -z "$TMPDIR" ]; then - BATS_TMPDIR="/tmp" -else - BATS_TMPDIR="${TMPDIR%/}" -fi - -BATS_TMPNAME="$BATS_TMPDIR/bats.$$" -BATS_PARENT_TMPNAME="$BATS_TMPDIR/bats.$PPID" +BATS_TMPDIR=$(getTmpDir) +BATS_TMPNAME=$(createTempFile $$ 0) # returns same result on each call +BATS_PARENT_TMPNAME=$(createTempFile $PPID 0) # ditto BATS_OUT="${BATS_TMPNAME}.out" bats_preprocess_source() { @@ -329,7 +355,12 @@ bats_evaluate_preprocessed_source() { source "$BATS_TEST_SOURCE" } -exec 3<&1 +# Set up the location of TAP output and location of run outputs +if [[ -n $BATS_OUTPUT_FILE ]]; then + exec 3>>$BATS_OUTPUT_FILE +else + exec 3<&1 # copy STDOUT file descriptor and use that instead of assuming STDOUT as the file descriptor of outputs +fi if [ "$#" -eq 0 ]; then bats_preprocess_source @@ -338,9 +369,9 @@ if [ "$#" -eq 0 ]; then if [ -n "$BATS_COUNT_ONLY" ]; then echo "${#BATS_TEST_NAMES[@]}" else - bats_perform_tests "${BATS_TEST_NAMES[@]}" + bats_perform_tests "${BATS_TEST_NAMES[@]}" fi else bats_evaluate_preprocessed_source - bats_perform_test "$@" + bats_perform_test "$@" fi diff --git a/libexec/bats-format-tap-stream b/libexec/bats-format-tap-stream index 614768f4..dc23acb9 100755 --- a/libexec/bats-format-tap-stream +++ b/libexec/bats-format-tap-stream @@ -144,7 +144,7 @@ while IFS= read -r line; do flush ;; "ok "* ) - skip_expr="ok $index # skip (\(([^)]*)\))?" + skip_expr="ok $index (.*) # skip ?(([^)]*))?" if [[ "$line" =~ $skip_expr ]]; then let skipped+=1 buffer skip "${BASH_REMATCH[2]}" diff --git a/libexec/common_functions.shrc b/libexec/common_functions.shrc new file mode 100644 index 00000000..41328211 --- /dev/null +++ b/libexec/common_functions.shrc @@ -0,0 +1,28 @@ + +function getTmpDir() { + local MYTMPDIR + if [ -z "$TMPDIR" ]; then + MYTMPDIR="/tmp" + else + MYTMPDIR="${TMPDIR%/}" + fi + + echo $MYTMPDIR +} + +# Generate temporary filename +function createTempFile() { + local PID="${1:-$$}" + local USETEMPLATE="${2:-1}" + local MYTMPDIR=$(getTmpDir) + + if [[ $USETEMPLATE == 1 ]]; then + # generate idempotent file names + TMPFILE=$(mktemp $MYTMPDIR/bats.$PID.XXXXXX) + echo $TMPFILE + else + # if we are not using a template we can call this function many times and get the same response + echo $MYTMPDIR/bats.$PID + fi +} + diff --git a/test/bats.bats b/test/bats.bats index f1aff293..08c50131 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -1,6 +1,7 @@ #!/usr/bin/env bats load test_helper +load "$BATS_PREFIX/libexec/common_functions.shrc" fixtures bats @test "no arguments prints usage instructions" { @@ -49,7 +50,16 @@ fixtures bats @test "summary passing and skipping tests" { run filter_control_sequences bats -p $FIXTURE_ROOT/passing_and_skipping.bats [ $status -eq 0 ] - [ "${lines[2]}" = "2 tests, 0 failures, 1 skipped" ] + [ "${lines[3]}" = "3 tests, 0 failures, 2 skipped" ] +} + +@test "tap passing and skipping tests" { + run filter_control_sequences bats --tap $FIXTURE_ROOT/passing_and_skipping.bats + [ $status -eq 0 ] + [ "${lines[0]}" = "1..3" ] + [ "${lines[1]}" = "ok 1 a passing test" ] + [ "${lines[2]}" = "ok 2 a skipping test # skip" ] + [ "${lines[3]}" = "ok 3 skip test with a reason # skip for a really good reason" ] } @test "summary passing and failing tests" { @@ -64,6 +74,15 @@ fixtures bats [ "${lines[5]}" = "3 tests, 1 failure, 1 skipped" ] } +@test "tap passing, failing and skipping tests" { + run filter_control_sequences bats --tap $FIXTURE_ROOT/passing_failing_and_skipping.bats + [ $status -eq 0 ] + [ "${lines[0]}" = "1..3" ] + [ "${lines[1]}" = "ok 1 a passing test" ] + [ "${lines[2]}" = "ok 2 a skipping test # skip" ] + [ "${lines[3]}" = "not ok 3 a failing test" ] +} + @test "one failing test" { run bats "$FIXTURE_ROOT/failing.bats" [ $status -eq 1 ] @@ -214,8 +233,8 @@ fixtures bats @test "skipped tests" { run bats "$FIXTURE_ROOT/skipped.bats" [ $status -eq 0 ] - [ "${lines[1]}" = "ok 1 # skip a skipped test" ] - [ "${lines[2]}" = "ok 2 # skip (a reason) a skipped test with a reason" ] + [ "${lines[1]}" = "ok 1 a skipped test # skip" ] + [ "${lines[2]}" = "ok 2 a skipped test with a reason # skip a reason" ] } @test "extended syntax" { @@ -262,3 +281,108 @@ fixtures bats [ $status -eq 0 ] [ "${lines[1]}" = "ok 1 loop_func" ] } + +@test "default test (in TAP)" { + run bats "$FIXTURE_ROOT/echo_output_and_pass.bats" + [ "$status" -eq 0 ] + + [ "${lines[0]}" = "1..2" ] + [ "${lines[1]}" = "ok 1 echo output and pass" ] + [ "${lines[2]}" = "ok 2 echo output and pass (using run)" ] +} + +@test "test with output released (in TAP)" { + run bats -r "$FIXTURE_ROOT/echo_output_and_pass.bats" + [ "$status" -eq 1 ] + + [ "${lines[0]}" = "ERROR: Cannot release output of tests to STDOUT and produce TAP output to STDOUT" ] +} + +@test "results directed to file (in TAP)" { + TMPFILE=$(createTempFile) + run bats -o $TMPFILE "$FIXTURE_ROOT/echo_output_and_pass.bats" + [ "$status" -eq 0 ] + + cat <