Skip to content

Commit

Permalink
Add information to logs with a custom runner
Browse files Browse the repository at this point in the history
Add a bash script to use as runner for our test suite

This script adds information to the logs:
- explicit names of the test run when they start
- some explicit messages for crashes (SIGSEGV, SIGBUS, ...)
- and, maybe most importantly, anchors in CI logs, so that the main
  webpage contains the most important information and direct links to
  precise points of interest in the log

It is written as a bash script but making sure it can be used as a
runner also on Windows CI (notably by commenting all ends of lines,
without which it fails with errors about '\r's that don't exist...)
  • Loading branch information
shym committed Nov 21, 2022
1 parent c4461ff commit 2ae5cb5
Show file tree
Hide file tree
Showing 20 changed files with 107 additions and 53 deletions.
6 changes: 3 additions & 3 deletions src/array/dune
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests)
Expand All @@ -32,7 +32,7 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -43,4 +43,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
6 changes: 3 additions & 3 deletions src/atomic/dune
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))


;; Linearizability tests of Atomic, utilizing ppx_deriving_qcheck
Expand All @@ -38,7 +38,7 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -49,4 +49,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
3 changes: 1 addition & 2 deletions src/bigarray/dune
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action
(run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
2 changes: 1 addition & 1 deletion src/buffer/dune
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
6 changes: 2 additions & 4 deletions src/bytes/dune
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,10 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action
(run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action
(run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
4 changes: 2 additions & 2 deletions src/domain/dune
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
(alias runtest)
(package multicoretests)
(deps domain_joingraph.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name domain_spawntree)
Expand All @@ -31,4 +31,4 @@
(alias runtest)
(deps domain_spawntree.exe)
(package multicoretests)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
8 changes: 4 additions & 4 deletions src/domainslib/dune
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
(alias runtest)
(package multicoretests)
(deps task_one_dep.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name task_more_deps)
Expand All @@ -34,7 +34,7 @@
(alias runtest)
(deps task_more_deps.exe)
(package multicoretests)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name task_parallel)
Expand All @@ -45,7 +45,7 @@
(alias runtest)
(package multicoretests)
(deps task_parallel.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))


;; STM_seq and STM_domain test of Domainslib.Chan
Expand All @@ -59,4 +59,4 @@
(rule
(alias runtest)
(deps chan_stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
4 changes: 2 additions & 2 deletions src/ephemeron/dune
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -26,4 +26,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
6 changes: 2 additions & 4 deletions src/floatarray/dune
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action
(run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -30,5 +29,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action
(run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
6 changes: 3 additions & 3 deletions src/hashtbl/dune
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests)
Expand All @@ -32,7 +32,7 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -43,4 +43,4 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
4 changes: 2 additions & 2 deletions src/internal/dune
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
(alias runtest)
(deps util_print_test.exe)
(package multicoretests)
(action (run ./%{deps})))
(action (bash "%{dep:../runner.sh} %{deps}")))


(executable
Expand All @@ -28,4 +28,4 @@
(alias runtest)
(deps cleanup.exe)
(package multicoretests)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
6 changes: 3 additions & 3 deletions src/lazy/dune
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests)
Expand All @@ -31,7 +31,7 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_dsl)
Expand All @@ -42,4 +42,4 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests_dsl.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))
2 changes: 1 addition & 1 deletion src/lockfree/dune
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@
(alias runtest)
(package multicoretests)
(deps ws_deque_test.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
22 changes: 11 additions & 11 deletions src/neg_tests/dune
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,19 @@
(alias runtest)
(package multicoretests)
(deps stm_tests_sequential_ref.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps stm_tests_domain_ref.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps stm_tests_thread_ref.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(library
(name CList)
Expand All @@ -73,7 +73,7 @@
(alias runtest)
(package multicoretests)
(deps stm_tests_conclist.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))


;; Linearizability tests of ref and Clist
Expand Down Expand Up @@ -112,19 +112,19 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl_domain.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

; (rule
; (alias runtest)
; (package multicoretests)
; (deps lin_tests_dsl_thread.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl_effect.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests_domain)
Expand All @@ -149,22 +149,22 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests_domain.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps lin_tests_thread_ref.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(rule
(alias runtest)
(package multicoretests)
(deps lin_tests_thread_conclist.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

; (rule
; (alias runtest)
; (package multicoretests)
; (deps lin_tests_effect.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))
4 changes: 2 additions & 2 deletions src/queue/dune
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
(alias runtest)
(package multicoretests)
(deps lin_tests_dsl.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))

(executable
(name lin_tests)
Expand All @@ -31,4 +31,4 @@
; (alias runtest)
; (package multicoretests)
; (deps lin_tests.exe)
; (action (run ./%{deps} --verbose)))
; (action (bash "%{dep:../runner.sh} %{deps}")))
59 changes: 59 additions & 0 deletions src/runner.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/bin/bash
# A runner for tests that adds timestamps and, in CI, anchors in logs
# All lines end as comments since bash on Windows CI seems to conjure
# some '\r' it doesn’t like into existence
test="$1" #
warning() { #
if [ "$CI" = true ] ; then #
printf '\n::warning title=%s in %s/%s::%s in src/%s/%s' "$1" "${PWD##*/}" "$test" "$1" "${PWD##*/}" "$test" #
else #
printf '\nWarning: %s in src/%s/%s' "$1" "${PWD##*/}" "$test" #
fi #
} #
error() { #
if [ "$CI" = true ] ; then #
printf '\n::error title=%s in %s/%s::%s in src/%s/%s' "$1" "${PWD##*/}" "$test" "$1" "${PWD##*/}" "$test" #
else #
printf '\nError: %s in src/%s/%s' "$1" "${PWD##*/}" "$test" #
fi #
} #
#
if [[ "$2" = "" ]] ; then #
printf '\n\nStarting "./%s --verbose" in %s\n' "$test" "${PWD##*/}" #
./"$test" --verbose #
result="$?" #
else #
shift #
printf '\n\nStarting "./%s %s" in %s\n' "$test" "$*" "${PWD##*/}" #
./"$test" "$@" #
result="$?" #
fi #
case "$result" in #
0) #
exit 0 #
;; #
1) #
warning "Test failure" #
exit 1 #
;; #
139) #
error SIGSEGV #
kill -SIGSEGV $$ #
;; #
135) #
error SIGBUS #
kill -SIGBUS $$ #
;; #
137) #
error SIGKILL #
kill -SIGKILL $$ #
;; #
134) #
error SIGABRT #
kill -SIGABRT $$ #
;; #
*) #
error "$result" #
exit "$result" #
;; #
esac #
2 changes: 1 addition & 1 deletion src/semaphore/dune
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@
(alias runtest)
(package multicoretests)
(deps stm_tests.exe)
(action (run ./%{deps} --verbose)))
(action (bash "%{dep:../runner.sh} %{deps}")))
Loading

0 comments on commit 2ae5cb5

Please sign in to comment.