From fe9f5be23bd4a07151dba9e64a320ad6b81b7620 Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 17:48:15 +0200 Subject: [PATCH 1/8] add doctest script --- doctest.sh | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100755 doctest.sh diff --git a/doctest.sh b/doctest.sh new file mode 100755 index 00000000..7fd3cf2d --- /dev/null +++ b/doctest.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# This file runs code snippets in doc/*.md files. + +set -e -o pipefail + +for arg in "$@"; do + if [[ "$arg" =~ ^- ]]; then + echo "Usage: $0 [doc/file1.md doc/file2.md ...]" >&2 + exit 2 + fi +done + +if [ $# == 0 ]; then + files=(doc/*.md) +else + files=("$@") +fi + +if [[ "$OS" =~ Windows ]]; then + source activate + mingw32-make + jou="$PWD/jou.exe" +else + make + jou="$PWD/jou" +fi + +function slice() +{ + local first_lineno="$1" + local last_lineno="$2" + local num_lines=$((last_lineno - first_lineno + 1)) + head -n $last_lineno | tail -n $num_lines +} + +function generate_expected_output() +{ + local joufile="$1" + + (grep -onH '# Warning: .*' "$joufile" || true) | sed -E s/'(.*):([0-9]*):# Warning: '/'compiler warning for file "test.jou", line \2: '/ + (grep -onH '# Error: .*' "$joufile" || true) | sed -E s/'(.*):([0-9]*):# Error: '/'compiler error in file "\1", line \2: '/ + (grep -oE '# Output:.*' "$joufile" || true) | sed -E s/'^# Output: ?'// +} + +rm -rf tmp/doctest +mkdir tmp/doctest + +for file in "${files[@]}"; do + echo "Extracting doctests from $file..." + mkdir tmp/doctest/"$(basename "$file")" + + for start_marker_lineno in $(grep -n '^```python$' "$file" | cut -d: -f1); do + outfile="tmp/doctest/$(basename "$file")/$((start_marker_lineno + 1)).jou" + awk -v n=$start_marker_lineno '(/^```$/ && line > n) { stop=1 } (++line > n && !stop) { print }' "$file" > "$outfile" + + # Do not test if there is no expected output/errors + if [ -z "$(generate_expected_output "$outfile")" ]; then + rm "$outfile" + fi + done +done + +ntotal=$(ls -1 tmp/doctest/*/*.jou | wc -l) +if [ $ntotal == 0 ]; then + echo "*** Error: no doctests found ***" >&2 + exit 1 +fi +echo "Running $ntotal doctests..." +nfail=0 + +cd tmp/doctest +for file in */*.jou; do + echo "${file%.*}" | tr '/' ':' # foo.md/123.jou --> foo.md:123 + cp "$file" test.jou + if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( "$jou" test.jou 2>&1 || true | tr -d '\r'); then + echo " ok" + else + ((nfail++)) || true + fi +done + +echo "" +echo "" + +echo "$((ntotal-nfail)) succeeded, $nfail failed" +if [ $nfail != 0 ]; then + exit 1 +fi From de30eeb842b1de3fd50ee2bb585260ea1908e095 Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 18:32:08 +0200 Subject: [PATCH 2/8] improve wrap around example --- doc/perf.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/doc/perf.md b/doc/perf.md index 4c2d52cd..6b3e2144 100644 --- a/doc/perf.md +++ b/doc/perf.md @@ -279,23 +279,25 @@ The takeaway from this is that these are all things that one would never do inte The rest of Jou's documentation aims to mention other things that are UB. In some other languages, it is easier to get UB than in Jou. -For example, in C it is UB to add two signed `int`s so large +For example, in C it is UB to add two `int`s so large that the result doesn't fit into an `int`, -but in Jou, math operations are guaranteed to "wrap around". -For example, Jou's `byte` is an unsigned 8-bit number, -so it has a range from 0 to 255, and bigger values wrap back around to 0: +but in Jou, math operations are instead guaranteed to wrap around: ```python -printf("%d\n", (255 as byte) + (1 as byte)) # Output: 0 -``` +import "stdlib/io.jou" -Here's what this looks like with `int`: +def main() -> int: + printf("%d\n", (254 as byte) + (0 as byte)) # Output: 254 + printf("%d\n", (254 as byte) + (1 as byte)) # Output: 255 + printf("%d\n", (254 as byte) + (2 as byte)) # Output: 0 + printf("%d\n", (254 as byte) + (3 as byte)) # Output: 1 + printf("%d\n", (254 as byte) + (4 as byte)) # Output: 2 + + printf("%d\n", 2147483646 + 0) # Output: 2147483646 + printf("%d\n", 2147483646 + 1) # Output: 2147483647 + printf("%d\n", 2147483646 + 2) # Output: -2147483648 + printf("%d\n", 2147483646 + 3) # Output: -2147483647 + printf("%d\n", 2147483646 + 4) # Output: -2147483646 -```python -printf("%d\n", 2147483647 + 1) # Output: -2147483648 + return 0 ``` - -The numbers are bigger, because `int` in Jou is 32 bits and `byte` is only 8 bits. -This time, the "wrapped around" result is negative, because `int` is signed. -In C this would be UB with a signed type (such as `int`), -but in Jou, overflowing integers is never UB. From efa880f5c72ed538b279a23e72dcb6ef44963f15 Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:16:12 +0200 Subject: [PATCH 3/8] document the documentation testing script --- CONTRIBUTING.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee50b014..ad0a5f83 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,8 +106,8 @@ Running tests (if on Windows, use Git Bash): $ ./runtests.sh ``` -This command does a few things: -- I compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. +The `runtests.sh` script does a few things: +- It compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. - It runs all Jou files in `examples/` and `tests/`. To speed things up, it runs two files in parallel. - It ensures that the Jou files output what is expected. @@ -165,6 +165,17 @@ This doesn't do anything with tests that are supposed to fail with an error, for There are also a few other ways to run the tests. You can look at `.github/workflows/` to see how the CI runs tests. +To ensure that documentation stays up to date, +it is also possible to run code examples in the documentation as tests: + +``` +$ ./doctest.sh +``` + +The `doctest.sh` script finds code examples from markdown files in `doc/`. +It only looks at code examples that contain `# Output:`, `# Warning:` or `# Error:` comments. +It then attempts to run each example and compares the output similarly to `runtests.sh`. + ## Windows Release Builds From 3d5f60e0d7e9e097fff9cdb8fc4f1ff34601d21f Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:28:06 +0200 Subject: [PATCH 4/8] run doctest in ci --- .github/workflows/linux.yml | 11 +++++++++++ .github/workflows/macos.yml | 8 ++++++++ .github/workflows/windows.yml | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b89c555c..780f6229 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -47,6 +47,17 @@ jobs: exit 1 fi + doctest: + runs-on: ubuntu-latest + strategy: + matrix: + llvm-version: [11, 13] + steps: + - uses: actions/checkout@v3 + - run: sudo apt install -y llvm-${{ matrix.llvm-version }}-dev clang-${{ matrix.llvm-version }} make + - run: LLVM_CONFIG=llvm-config-${{ matrix.llvm-version }} make + - run: ./doctest.sh + compare-compilers: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2a1e2e3d..72bad46c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -26,6 +26,14 @@ jobs: exit 1 fi + doctest: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - run: brew install bash diffutils llvm@13 + - run: make + - run: ./doctest.sh + compare-compilers: runs-on: macos-latest steps: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 435b6316..1f959b83 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -157,6 +157,15 @@ jobs: - run: cd "test dir" && source activate && ./runtests.sh --verbose shell: bash + doctest: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - run: ./windows_setup.sh --small + shell: bash + - run: source activate && ./doctest.sh + shell: bash + test-zip: needs: build-zip runs-on: windows-latest From d6abcaf8f562fde4f9f31dd69588698343c1b9eb Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:32:51 +0200 Subject: [PATCH 5/8] try to satisfy shellcheck --- .github/workflows/linux.yml | 2 +- doctest.sh | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 780f6229..ead768f5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: shellcheck --color=always --shell=bash --exclude=SC2086,SC2059,SC2046,SC2235,SC2002,SC2206,SC2068,SC2207 *.sh activate + - run: shellcheck --color=always --shell=bash --exclude=SC2086,SC2059,SC2046,SC2235,SC2002,SC2206,SC2068,SC2207,SC2013 *.sh activate test: runs-on: ubuntu-latest diff --git a/doctest.sh b/doctest.sh index 7fd3cf2d..6c0bdf0f 100755 --- a/doctest.sh +++ b/doctest.sh @@ -61,12 +61,7 @@ for file in "${files[@]}"; do done done -ntotal=$(ls -1 tmp/doctest/*/*.jou | wc -l) -if [ $ntotal == 0 ]; then - echo "*** Error: no doctests found ***" >&2 - exit 1 -fi -echo "Running $ntotal doctests..." +ntotal=0 nfail=0 cd tmp/doctest @@ -76,10 +71,16 @@ for file in */*.jou; do if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( "$jou" test.jou 2>&1 || true | tr -d '\r'); then echo " ok" else - ((nfail++)) || true + ((nfail++)) fi + ((ntotal++)) done +if [ $ntotal == 0 ]; then + echo "*** Error: no doctests found ***" >&2 + exit 1 +fi + echo "" echo "" From 3378a9aa4bf2df2ebc5200699232433db0cde4fb Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:34:56 +0200 Subject: [PATCH 6/8] p --- doctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctest.sh b/doctest.sh index 6c0bdf0f..4035bde9 100755 --- a/doctest.sh +++ b/doctest.sh @@ -44,7 +44,7 @@ function generate_expected_output() } rm -rf tmp/doctest -mkdir tmp/doctest +mkdir -p tmp/doctest for file in "${files[@]}"; do echo "Extracting doctests from $file..." From 5d64cf0f8dcb50eec49eb41024f523bd5b36ebf1 Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:35:43 +0200 Subject: [PATCH 7/8] wtf bash --- doctest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doctest.sh b/doctest.sh index 4035bde9..f34c277c 100755 --- a/doctest.sh +++ b/doctest.sh @@ -71,9 +71,9 @@ for file in */*.jou; do if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( "$jou" test.jou 2>&1 || true | tr -d '\r'); then echo " ok" else - ((nfail++)) + ((nfail++)) || true fi - ((ntotal++)) + ((ntotal++)) || true done if [ $ntotal == 0 ]; then From 93379bf55e202edf3be008ed5a15c9abe416350e Mon Sep 17 00:00:00 2001 From: Akuli Date: Thu, 7 Dec 2023 19:44:17 +0200 Subject: [PATCH 8/8] fix? --- doctest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctest.sh b/doctest.sh index f34c277c..05c99791 100755 --- a/doctest.sh +++ b/doctest.sh @@ -68,7 +68,7 @@ cd tmp/doctest for file in */*.jou; do echo "${file%.*}" | tr '/' ':' # foo.md/123.jou --> foo.md:123 cp "$file" test.jou - if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( "$jou" test.jou 2>&1 || true | tr -d '\r'); then + if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( ("$jou" test.jou 2>&1 || true) | tr -d '\r'); then echo " ok" else ((nfail++)) || true