diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5064aa62..4237086c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,9 +15,9 @@ jobs: - uses: actions/checkout@v3 - run: sudo apt install -y llvm-${{ matrix.llvm-version }}-dev clang-${{ matrix.llvm-version }} make valgrind - run: LLVM_CONFIG=llvm-config-${{ matrix.llvm-version }} make - - run: ./runtests.sh './jou ${{ matrix.opt-level }} %s' - - run: ./runtests.sh './jou ${{ matrix.opt-level }} --verbose %s' - - run: ./runtests.sh --valgrind './jou ${{ matrix.opt-level }} %s' + - run: ./runtests.sh --verbose './jou ${{ matrix.opt-level }} %s' + - run: ./runtests.sh --verbose './jou ${{ matrix.opt-level }} --verbose %s' + - run: ./runtests.sh --verbose --valgrind './jou ${{ matrix.opt-level }} %s' # valgrind+verbose isn't meaningful: test script would ignore valgrind output fuzzer: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 67b086b7..53d4893a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -126,7 +126,7 @@ jobs: shell: bash - run: cd "test dir" && ./jou.exe --verbose examples/hello.jou shell: bash - - run: cd "test dir" && ./runtests.sh + - run: cd "test dir" && ./runtests.sh --verbose shell: bash fuzzer: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f45f5e0c..bf8c5003 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,7 +101,7 @@ $ ./runtests.sh This command does a few things: - If not on Windows, it compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. (On Windows you need to use the build button in CodeBlocks to compile.) -- It runs all Jou files in `examples/` and `tests/` +- It runs all Jou files in `examples/` and `tests/`. To speed things up, it runs two files in parallel. - It ensures that the Jou files output what is expected. The expected output is auto-generated from comments in the Jou files: diff --git a/runtests.sh b/runtests.sh index b32539c9..da0c7800 100755 --- a/runtests.sh +++ b/runtests.sh @@ -11,12 +11,22 @@ export LANG=C # "Segmentation fault" must be in english for this script to work set -e -o pipefail -if [ "$1" == "--valgrind" ]; then - valgrind=yes - shift -else - valgrind=no -fi +function usage() { + echo "Usage: $0 [--valgrind] [--verbose] [TEMPLATE]" >&2 + echo "TEMPLATE can be e.g. './jou %s', where %s will be replaced by a jou file." >&2 + exit 2 +} + +valgrind=no +verbose=no + +while [[ "$1" =~ ^- ]]; do + case "$1" in + --valgrind) valgrind=yes; shift ;; + --verbose) verbose=yes; shift ;; + *) usage ;; + esac +done if [ $# == 0 ]; then # No arguments --> run tests in the basic/simple way @@ -25,12 +35,10 @@ if [ $# == 0 ]; then else command_template='./jou %s' fi -elif [ $# == 1 ] && [[ "$1" =~ ^[^-] ]]; then +elif [ $# == 1 ]; then command_template="$1" else - echo "Usage: $0 [--valgrind] [TEMPLATE]" >&2 - echo "TEMPLATE can be e.g. './jou %s', where %s will be replaced by a jou file." >&2 - exit 2 + usage fi if [ $valgrind = yes ]; then @@ -106,6 +114,18 @@ GREEN="\x1b[32m" RED="\x1b[31m" RESET="\x1b[0m" +if [ $verbose = yes ]; then + function show_run() { echo "run: $1"; } + function show_skip() { echo -e "${YELLOW}skip${RESET} $1"; } + function show_ok() { echo -e "${GREEN}ok${RESET} $1"; } + function show_fail() { echo -e "${RED}FAIL${RESET} $1"; } +else + function show_run() { true; } + function show_skip() { echo -ne ${YELLOW}s${RESET}; } + function show_ok() { echo -ne ${GREEN}.${RESET}; } + function show_fail() { echo -ne ${RED}F${RESET}; } +fi + function run_test() { local joufile="$1" @@ -122,20 +142,22 @@ function run_test() if ( [[ "$command_template" =~ -O[1-3] ]] && [[ $joufile =~ ^tests/crash/ ]] ) \ || ( [[ "$command_template" =~ valgrind ]] && [ $correct_exit_code != 0 ] ) then - # Skip - echo -ne ${YELLOW}s${RESET} + show_skip $joufile mv $diffpath $diffpath.skip - elif diff -u --color=always \ + return + fi + + show_run $joufile + if diff -u --color=always \ <(generate_expected_output $joufile $correct_exit_code | tr -d '\r') \ <(ulimit -v 500000 2>/dev/null; bash -c "$command; echo Exit code: \$?" 2>&1 | post_process_output $joufile | tr -d '\r') \ &>> $diffpath then - # Ran successfully - echo -ne ${GREEN}.${RESET} + show_ok $joufile rm -f $diffpath else - # Failed - echo -ne ${RED}F${RESET} + show_fail $joufile + # Do not delete diff file. It will be displayed at end of test run. fi }