From 4583a18a2f2e0495d77a317ec3336ca5fd1747ff Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Wed, 20 Apr 2016 12:12:16 +0200 Subject: [PATCH 1/9] Add a JUnit output formatter compatible with Jenkins --- libexec/bats | 15 ++++- libexec/bats-exec-suite | 3 + libexec/bats-format-junit | 113 ++++++++++++++++++++++++++++++++++++++ test/suite.bats | 14 +++-- 4 files changed, 137 insertions(+), 8 deletions(-) create mode 100755 libexec/bats-format-junit diff --git a/libexec/bats b/libexec/bats index 71f392f7..4c4a031a 100755 --- a/libexec/bats +++ b/libexec/bats @@ -20,6 +20,8 @@ help() { echo " -h, --help Display this help message" echo " -p, --pretty Show results in pretty format (default for terminals)" echo " -t, --tap Show results in TAP format" + echo " -x, --extended Show results in extended TAP format" + echo " -u, --junit Show results in JUnit format, hudson compatible" echo " -v, --version Display the version number" echo echo " For more information, see https://github.com/sstephenson/bats" @@ -93,12 +95,19 @@ for option in "${options[@]}"; do "c" | "count" ) count_flag="-c" ;; + "x" | "extended" ) + pretty="" + extended_syntax_flag="-x" + ;; "t" | "tap" ) pretty="" ;; "p" | "pretty" ) pretty="1" ;; + "u" | "junit" ) + junit="1" + ;; * ) usage >&2 exit 1 @@ -130,11 +139,13 @@ else command="bats-exec-suite" fi -if [ -n "$pretty" ]; then +if [ -n "$junit" ]; then + extended_syntax_flag="-x" + formatter="bats-format-junit" +elif [ -n "$pretty" ]; then extended_syntax_flag="-x" formatter="bats-format-tap-stream" else - extended_syntax_flag="" formatter="cat" fi diff --git a/libexec/bats-exec-suite b/libexec/bats-exec-suite index 29ab255d..b75450fe 100755 --- a/libexec/bats-exec-suite +++ b/libexec/bats-exec-suite @@ -30,6 +30,9 @@ status=0 offset=0 for filename in "$@"; do index=0 + if test -n "$extended_syntax_flag"; then + echo suite bats.$(basename "$filename"| sed -e 's/\.bats//') + fi { IFS= read -r # 1..n while IFS= read -r line; do diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit new file mode 100755 index 00000000..c3158f46 --- /dev/null +++ b/libexec/bats-format-junit @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -e + +header_pattern='[0-9]+\.\.[0-9]+' +IFS= read -r header + +if [[ "$header" =~ $header_pattern ]]; then + count="${header:3}" + index=0 + failures=0 + skipped=0 + name="" +else + # If the first line isn't a TAP plan, print it and pass the rest through + printf "%s\n" "$header" + exec cat +fi + +header() { + printf "\ + +\n" +} + +footer() { + printf "\n" +} + + +pass() { + echo " \n" +} + +fail() { + echo "\ + + + \n" +} + +skip() { + echo "\ + + $1 + \n" +} + +_buffer="" + +buffer() { + _buffer="${_buffer}$("$@")" +} + +flush() { + printf "$_buffer" + _buffer="" +} + +_buffer_log="" +log() { + _buffer_log="${_buffer_log}\n$1" +} + +flush_log() { + if [[ -n "${_buffer_log}" ]]; then + buffer fail "${_buffer_log}" + _buffer_log="" + fi +} + +finish() { + flush_log + header + flush + footer +} + +trap finish EXIT + + +while IFS= read -r line; do + case "$line" in + "suite "*) + flush_log + suite_expr="suite (.*)" + if [[ "$line" =~ $suite_expr ]]; then + class="${BASH_REMATCH[1]}" + fi + ;; + "begin "* ) + flush_log + let index+=1 + name="${line#* $index }" + ;; + "ok "* ) + skip_expr="ok $index # skip (\(([^)]*)\))?" + if [[ "$line" =~ $skip_expr ]]; then + let skipped+=1 + buffer skip "${BASH_REMATCH[2]}" + else + buffer pass + fi + ;; + "not ok "* ) + let failures+=1 + ;; + "# "* ) + log "${line:2}" + ;; + esac +done + + diff --git a/test/suite.bats b/test/suite.bats index 53716863..317e38fd 100755 --- a/test/suite.bats +++ b/test/suite.bats @@ -55,10 +55,12 @@ fixtures suite FLUNK=1 run bats-exec-suite -x "$FIXTURE_ROOT/multiple/"*.bats [ $status -eq 1 ] [ "${lines[0]}" = "1..3" ] - [ "${lines[1]}" = "begin 1 truth" ] - [ "${lines[2]}" = "ok 1 truth" ] - [ "${lines[3]}" = "begin 2 more truth" ] - [ "${lines[4]}" = "ok 2 more truth" ] - [ "${lines[5]}" = "begin 3 quasi-truth" ] - [ "${lines[6]}" = "not ok 3 quasi-truth" ] + [ "${lines[1]}" = "suite bats.a" ] + [ "${lines[2]}" = "begin 1 truth" ] + [ "${lines[3]}" = "ok 1 truth" ] + [ "${lines[4]}" = "suite bats.b" ] + [ "${lines[5]}" = "begin 2 more truth" ] + [ "${lines[6]}" = "ok 2 more truth" ] + [ "${lines[7]}" = "begin 3 quasi-truth" ] + [ "${lines[8]}" = "not ok 3 quasi-truth" ] } From f684ed4976fdb3680c0c95b48a64a4f684d0d41d Mon Sep 17 00:00:00 2001 From: Julien Cornebise Date: Wed, 11 May 2016 17:34:01 +0100 Subject: [PATCH 2/9] Color passing tests in green This is in line with the *unit tests, where we can see red or green, not just red. Just my two cents :) --- libexec/bats-format-tap-stream | 1 + 1 file changed, 1 insertion(+) diff --git a/libexec/bats-format-tap-stream b/libexec/bats-format-tap-stream index 614768f4..c5f7a443 100755 --- a/libexec/bats-format-tap-stream +++ b/libexec/bats-format-tap-stream @@ -39,6 +39,7 @@ begin() { pass() { go_to_column 0 + set_color 2 printf " ✓ %s" "$name" advance } From b22040113d48c17bd53fcec293f3d57e3e3f23e8 Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Tue, 19 Jul 2016 17:43:46 +0200 Subject: [PATCH 3/9] Add the time in seconds elapsed by each test in extended format and in junit report --- libexec/bats-exec-test | 8 +++++++- libexec/bats-format-junit | 31 +++++++++++++++++-------------- test/bats.bats | 2 +- test/suite.bats | 4 ++-- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 8f3bd510..7aea87dd 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -81,6 +81,7 @@ skip() { bats_test_begin() { BATS_TEST_DESCRIPTION="$1" + BATS_TEST_START_TIME=${SECONDS} if [ -n "$BATS_EXTENDED_SYNTAX" ]; then echo "begin $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 fi @@ -259,7 +260,12 @@ bats_exit_trap() { sed -e "s/^/# /" < "$BATS_OUT" >&3 status=1 else - echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 + if [ -n "${skipped}" ] || [ -z "$BATS_EXTENDED_SYNTAX" ]; then + BATS_TEST_TIME="" + else + BATS_TEST_TIME=" in "$((SECONDS - BATS_TEST_START_TIME))"sec" + fi + echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3 status=0 fi diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit index c3158f46..fcc32756 100755 --- a/libexec/bats-format-junit +++ b/libexec/bats-format-junit @@ -28,19 +28,19 @@ footer() { pass() { - echo " \n" + echo " \n" } fail() { echo "\ - + \n" } skip() { echo "\ - + $1 \n" } @@ -63,21 +63,20 @@ log() { flush_log() { if [[ -n "${_buffer_log}" ]]; then - buffer fail "${_buffer_log}" - _buffer_log="" + buffer fail "${_buffer_log}" + _buffer_log="" fi } finish() { - flush_log - header - flush - footer + flush_log + header + flush + footer } trap finish EXIT - while IFS= read -r line; do case "$line" in "suite "*) @@ -93,12 +92,17 @@ while IFS= read -r line; do name="${line#* $index }" ;; "ok "* ) - skip_expr="ok $index # skip (\(([^)]*)\))?" - if [[ "$line" =~ $skip_expr ]]; then + expr_ok="ok $index (.*) in ([0-9]+)sec" + expr_skip="ok $index # skip (.*)" + if [[ "$line" =~ $expr_skip ]]; then let skipped+=1 buffer skip "${BASH_REMATCH[2]}" - else + elif [[ "$line" =~ $expr_ok ]]; then + test_exec_time="${BASH_REMATCH[2]}" buffer pass + else + log "Wrong output format: ${line}" + let failures+=1 fi ;; "not ok "* ) @@ -110,4 +114,3 @@ while IFS= read -r line; do esac done - diff --git a/test/bats.bats b/test/bats.bats index f1aff293..b5e6d949 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -224,7 +224,7 @@ fixtures bats [ "${lines[1]}" = 'begin 1 a failing test' ] [ "${lines[2]}" = 'not ok 1 a failing test' ] [ "${lines[5]}" = 'begin 2 a passing test' ] - [ "${lines[6]}" = 'ok 2 a passing test' ] + [ "${lines[6]}" = 'ok 2 a passing test in 0sec' ] } @test "pretty and tap formats" { diff --git a/test/suite.bats b/test/suite.bats index 317e38fd..9610c1a3 100755 --- a/test/suite.bats +++ b/test/suite.bats @@ -57,10 +57,10 @@ fixtures suite [ "${lines[0]}" = "1..3" ] [ "${lines[1]}" = "suite bats.a" ] [ "${lines[2]}" = "begin 1 truth" ] - [ "${lines[3]}" = "ok 1 truth" ] + [ "${lines[3]}" = "ok 1 truth in 0sec" ] [ "${lines[4]}" = "suite bats.b" ] [ "${lines[5]}" = "begin 2 more truth" ] - [ "${lines[6]}" = "ok 2 more truth" ] + [ "${lines[6]}" = "ok 2 more truth in 0sec" ] [ "${lines[7]}" = "begin 3 quasi-truth" ] [ "${lines[8]}" = "not ok 3 quasi-truth" ] } From 2527d22db0c19f6e34f58a4550637953f411b466 Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Wed, 20 Jul 2016 16:02:39 +0200 Subject: [PATCH 4/9] Add the ability to provide bats pathes from environment --- libexec/bats | 6 +++--- test/bats.bats | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libexec/bats b/libexec/bats index 4c4a031a..0aea301c 100755 --- a/libexec/bats +++ b/libexec/bats @@ -54,9 +54,9 @@ expand_path() { } || echo "$1" } -BATS_LIBEXEC="$(abs_dirname "$0")" -export BATS_PREFIX="$(abs_dirname "$BATS_LIBEXEC")" -export BATS_CWD="$(abs_dirname .)" +BATS_LIBEXEC="${BATS_LIBEXEC-$(abs_dirname "$0")}" +export BATS_PREFIX="${BATS_PREFIX-$(abs_dirname "$BATS_LIBEXEC")}" +export BATS_CWD="${BATS_CWD-$(abs_dirname .)}" export PATH="$BATS_LIBEXEC:$PATH" options=() diff --git a/test/bats.bats b/test/bats.bats index b5e6d949..b9c7d07d 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -150,6 +150,7 @@ fixtures bats } @test "failing test file outside of BATS_CWD" { + unset BATS_LIBEXEC BATS_PREFIX BATS_CWD cd "$TMP" run bats "$FIXTURE_ROOT/failing.bats" [ $status -eq 1 ] From fdbc7d8d0bd2a5b8e9c7fdba1147cb68fd123563 Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Thu, 21 Jul 2016 10:00:49 +0200 Subject: [PATCH 5/9] Add the total execution time to the testsuite --- libexec/bats-format-junit | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit index fcc32756..aff0411a 100755 --- a/libexec/bats-format-junit +++ b/libexec/bats-format-junit @@ -9,6 +9,7 @@ if [[ "$header" =~ $header_pattern ]]; then index=0 failures=0 skipped=0 + suitetest_exec_time=0 name="" else # If the first line isn't a TAP plan, print it and pass the rest through @@ -19,7 +20,7 @@ fi header() { printf "\ -\n" +\n" } footer() { @@ -99,6 +100,7 @@ while IFS= read -r line; do buffer skip "${BASH_REMATCH[2]}" elif [[ "$line" =~ $expr_ok ]]; then test_exec_time="${BASH_REMATCH[2]}" + suitetest_exec_time=$((suitetest_exec_time + test_exec_time)) buffer pass else log "Wrong output format: ${line}" From 118b5f056aeb10ddb77cac70af6b0032bf8253cb Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Thu, 21 Jul 2016 10:58:47 +0200 Subject: [PATCH 6/9] Output each testcase report into a specific XML file --- libexec/bats-format-junit | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit index aff0411a..04cf8b2a 100755 --- a/libexec/bats-format-junit +++ b/libexec/bats-format-junit @@ -4,23 +4,21 @@ set -e header_pattern='[0-9]+\.\.[0-9]+' IFS= read -r header -if [[ "$header" =~ $header_pattern ]]; then - count="${header:3}" - index=0 +index=0 + +init_suite() { + count=0 failures=0 skipped=0 suitetest_exec_time=0 name="" -else - # If the first line isn't a TAP plan, print it and pass the rest through - printf "%s\n" "$header" - exec cat -fi + _buffer="" +} header() { printf "\ -\n" +\n" } footer() { @@ -46,8 +44,6 @@ skip() { \n" } -_buffer="" - buffer() { _buffer="${_buffer}$("$@")" } @@ -69,19 +65,25 @@ flush_log() { fi } -finish() { - flush_log - header - flush - footer +finish_suite() { + [[ ${count} -gt 0 ]] && { + ( + flush_log + header + flush + footer + ) > "TestReport-${class}.xml" + } + init_suite } -trap finish EXIT +trap finish_suite EXIT while IFS= read -r line; do case "$line" in "suite "*) flush_log + finish_suite suite_expr="suite (.*)" if [[ "$line" =~ $suite_expr ]]; then class="${BASH_REMATCH[1]}" @@ -93,6 +95,7 @@ while IFS= read -r line; do name="${line#* $index }" ;; "ok "* ) + let count+=1 expr_ok="ok $index (.*) in ([0-9]+)sec" expr_skip="ok $index # skip (.*)" if [[ "$line" =~ $expr_skip ]]; then @@ -108,6 +111,7 @@ while IFS= read -r line; do fi ;; "not ok "* ) + let count+=1 let failures+=1 ;; "# "* ) From 2377aac9f8d81e3d639560b21bd5cd5a9c61c371 Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Thu, 21 Jul 2016 11:19:53 +0200 Subject: [PATCH 7/9] Add execution time in case of error --- libexec/bats-exec-test | 13 +++++++------ libexec/bats-format-junit | 17 +++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 7aea87dd..ca407c56 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -253,18 +253,19 @@ bats_exit_trap() { fi fi + if [ -n "${skipped}" ] || [ -z "$BATS_EXTENDED_SYNTAX" ]; then + BATS_TEST_TIME="" + else + BATS_TEST_TIME=" in "$((SECONDS - BATS_TEST_START_TIME))"sec" + fi + if [ -z "$BATS_TEST_COMPLETED" ] || [ -z "$BATS_TEARDOWN_COMPLETED" ]; then - echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3 + echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION${BATS_TEST_TIME}" >&3 bats_print_stack_trace "${BATS_ERROR_STACK_TRACE[@]}" >&3 bats_print_failed_command "${BATS_ERROR_STACK_TRACE[${#BATS_ERROR_STACK_TRACE[@]}-1]}" "$BATS_ERROR_STATUS" >&3 sed -e "s/^/# /" < "$BATS_OUT" >&3 status=1 else - if [ -n "${skipped}" ] || [ -z "$BATS_EXTENDED_SYNTAX" ]; then - BATS_TEST_TIME="" - else - BATS_TEST_TIME=" in "$((SECONDS - BATS_TEST_START_TIME))"sec" - fi echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}${BATS_TEST_TIME}" >&3 status=0 fi diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit index 04cf8b2a..dd6c4530 100755 --- a/libexec/bats-format-junit +++ b/libexec/bats-format-junit @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -e -header_pattern='[0-9]+\.\.[0-9]+' IFS= read -r header index=0 @@ -32,7 +31,7 @@ pass() { fail() { echo "\ - + \n" } @@ -72,7 +71,7 @@ finish_suite() { header flush footer - ) > "TestReport-${class}.xml" + ) > "TestReport-${class-case}.xml" } init_suite } @@ -96,13 +95,14 @@ while IFS= read -r line; do ;; "ok "* ) let count+=1 - expr_ok="ok $index (.*) in ([0-9]+)sec" + expr_ok="ok $index .* in ([0-9]+)sec" expr_skip="ok $index # skip (.*)" if [[ "$line" =~ $expr_skip ]]; then let skipped+=1 - buffer skip "${BASH_REMATCH[2]}" + test_exec_time=0 + buffer skip "${BASH_REMATCH[1]}" elif [[ "$line" =~ $expr_ok ]]; then - test_exec_time="${BASH_REMATCH[2]}" + test_exec_time="${BASH_REMATCH[1]}" suitetest_exec_time=$((suitetest_exec_time + test_exec_time)) buffer pass else @@ -113,6 +113,11 @@ while IFS= read -r line; do "not ok "* ) let count+=1 let failures+=1 + expr_notok="not ok $index .* in ([0-9]+)sec" + if [[ "$line" =~ $expr_notok ]]; then + test_exec_time="${BASH_REMATCH[1]}" + suitetest_exec_time=$((suitetest_exec_time + test_exec_time)) + fi ;; "# "* ) log "${line:2}" From 45ec816cffab8dde42d73a3f331c78142e4add6e Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Thu, 4 Aug 2016 14:54:03 +0200 Subject: [PATCH 8/9] bugfix: use echo instead of printf for user input, this avoid printf escaping errors --- libexec/bats-format-junit | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/libexec/bats-format-junit b/libexec/bats-format-junit index dd6c4530..c491b54a 100755 --- a/libexec/bats-format-junit +++ b/libexec/bats-format-junit @@ -30,9 +30,13 @@ pass() { } fail() { - echo "\ + printf "\ - + \n" } @@ -48,13 +52,14 @@ buffer() { } flush() { - printf "$_buffer" + echo "${_buffer}" _buffer="" } _buffer_log="" log() { - _buffer_log="${_buffer_log}\n$1" + _buffer_log="${_buffer_log} +$1" } flush_log() { From c57535435ed076dc3dc3b0d42c353a78275530c5 Mon Sep 17 00:00:00 2001 From: Camille Meulien Date: Thu, 4 Aug 2016 15:16:54 +0200 Subject: [PATCH 9/9] Fix tests --- test/bats.bats | 2 +- test/suite.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bats.bats b/test/bats.bats index b9c7d07d..187a834f 100755 --- a/test/bats.bats +++ b/test/bats.bats @@ -223,7 +223,7 @@ fixtures bats run bats-exec-test -x "$FIXTURE_ROOT/failing_and_passing.bats" [ $status -eq 1 ] [ "${lines[1]}" = 'begin 1 a failing test' ] - [ "${lines[2]}" = 'not ok 1 a failing test' ] + [ "${lines[2]}" = 'not ok 1 a failing test in 0sec' ] [ "${lines[5]}" = 'begin 2 a passing test' ] [ "${lines[6]}" = 'ok 2 a passing test in 0sec' ] } diff --git a/test/suite.bats b/test/suite.bats index 9610c1a3..fb2b875d 100755 --- a/test/suite.bats +++ b/test/suite.bats @@ -62,5 +62,5 @@ fixtures suite [ "${lines[5]}" = "begin 2 more truth" ] [ "${lines[6]}" = "ok 2 more truth in 0sec" ] [ "${lines[7]}" = "begin 3 quasi-truth" ] - [ "${lines[8]}" = "not ok 3 quasi-truth" ] + [ "${lines[8]}" = "not ok 3 quasi-truth in 0sec" ] }