Add tests for docs

.github/workflows/linux.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - run: shellcheck --color=always --shell=bash --exclude=SC2086,SC2059,SC2046,SC2235,SC2002,SC2206,SC2068,SC2207 *.sh activate
+      - run: shellcheck --color=always --shell=bash --exclude=SC2086,SC2059,SC2046,SC2235,SC2002,SC2206,SC2068,SC2207,SC2013 *.sh activate
 
   test:
     runs-on: ubuntu-latest
@@ -47,6 +47,17 @@ jobs:
             exit 1
           fi
 
+  doctest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        llvm-version: [11, 13]
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt install -y llvm-${{ matrix.llvm-version }}-dev clang-${{ matrix.llvm-version }} make
+      - run: LLVM_CONFIG=llvm-config-${{ matrix.llvm-version }} make
+      - run: ./doctest.sh
+
   compare-compilers:
     runs-on: ubuntu-latest
     steps:

.github/workflows/macos.yml

@@ -26,6 +26,14 @@ jobs:
             exit 1
           fi
 
+  doctest:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: brew install bash diffutils llvm@13
+      - run: make
+      - run: ./doctest.sh
+
   compare-compilers:
     runs-on: macos-latest
     steps:

.github/workflows/windows.yml

@@ -157,6 +157,15 @@ jobs:
       - run: cd "test dir" && source activate && ./runtests.sh --verbose
         shell: bash
 
+  doctest:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: ./windows_setup.sh --small
+        shell: bash
+      - run: source activate && ./doctest.sh
+        shell: bash
+
   test-zip:
     needs: build-zip
     runs-on: windows-latest

CONTRIBUTING.md

@@ -106,8 +106,8 @@ Running tests (if on Windows, use Git Bash):
 $ ./runtests.sh
 ```
 
-This command does a few things:
-- I compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled.
+The `runtests.sh` script does a few things:
+- It compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled.
 - It runs all Jou files in `examples/` and `tests/`. To speed things up, it runs two files in parallel.
 - It ensures that the Jou files output what is expected.
 
@@ -165,6 +165,17 @@ This doesn't do anything with tests that are supposed to fail with an error, for
 There are also a few other ways to run the tests.
 You can look at `.github/workflows/` to see how the CI runs tests.
 
+To ensure that documentation stays up to date,
+it is also possible to run code examples in the documentation as tests:
+
+```
+$ ./doctest.sh
+```
+
+The `doctest.sh` script finds code examples from markdown files in `doc/`.
+It only looks at code examples that contain `# Output:`, `# Warning:` or `# Error:` comments.
+It then attempts to run each example and compares the output similarly to `runtests.sh`.
+
 
 ## Windows Release Builds
 

doc/perf.md

@@ -279,23 +279,25 @@ The takeaway from this is that these are all things that one would never do inte
 The rest of Jou's documentation aims to mention other things that are UB.
 
 In some other languages, it is easier to get UB than in Jou.
-For example, in C it is UB to add two signed `int`s so large
+For example, in C it is UB to add two `int`s so large
 that the result doesn't fit into an `int`,
-but in Jou, math operations are guaranteed to "wrap around".
-For example, Jou's `byte` is an unsigned 8-bit number,
-so it has a range from 0 to 255, and bigger values wrap back around to 0:
+but in Jou, math operations are instead guaranteed to wrap around:
 
 ```python
-printf("%d\n", (255 as byte) + (1 as byte))   # Output: 0
-```
+import "stdlib/io.jou"
 
-Here's what this looks like with `int`:
+def main() -> int:
+    printf("%d\n", (254 as byte) + (0 as byte))   # Output: 254
+    printf("%d\n", (254 as byte) + (1 as byte))   # Output: 255
+    printf("%d\n", (254 as byte) + (2 as byte))   # Output: 0
+    printf("%d\n", (254 as byte) + (3 as byte))   # Output: 1
+    printf("%d\n", (254 as byte) + (4 as byte))   # Output: 2
+
+    printf("%d\n", 2147483646 + 0)   # Output: 2147483646
+    printf("%d\n", 2147483646 + 1)   # Output: 2147483647
+    printf("%d\n", 2147483646 + 2)   # Output: -2147483648
+    printf("%d\n", 2147483646 + 3)   # Output: -2147483647
+    printf("%d\n", 2147483646 + 4)   # Output: -2147483646
 
-```python
-printf("%d\n", 2147483647 + 1)   # Output: -2147483648
+    return 0
 ```
-
-The numbers are bigger, because `int` in Jou is 32 bits and `byte` is only 8 bits.
-This time, the "wrapped around" result is negative, because `int` is signed.
-In C this would be UB with a signed type (such as `int`),
-but in Jou, overflowing integers is never UB.

doctest.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+#
+# This file runs code snippets in doc/*.md files.
+
+set -e -o pipefail
+
+for arg in "$@"; do
+    if [[ "$arg" =~ ^- ]]; then
+        echo "Usage: $0 [doc/file1.md doc/file2.md ...]" >&2
+        exit 2
+    fi
+done
+
+if [ $# == 0 ]; then
+    files=(doc/*.md)
+else
+    files=("$@")
+fi
+
+if [[ "$OS" =~ Windows ]]; then
+    source activate
+    mingw32-make
+    jou="$PWD/jou.exe"
+else
+    make
+    jou="$PWD/jou"
+fi
+
+function slice()
+{
+    local first_lineno="$1"
+    local last_lineno="$2"
+    local num_lines=$((last_lineno - first_lineno + 1))
+    head -n $last_lineno | tail -n $num_lines
+}
+
+function generate_expected_output()
+{
+    local joufile="$1"
+
+    (grep -onH '# Warning: .*' "$joufile" || true) | sed -E s/'(.*):([0-9]*):# Warning: '/'compiler warning for file "test.jou", line \2: '/
+    (grep -onH '# Error: .*' "$joufile" || true) | sed -E s/'(.*):([0-9]*):# Error: '/'compiler error in file "\1", line \2: '/
+    (grep -oE '# Output:.*' "$joufile" || true) | sed -E s/'^# Output: ?'//
+}
+
+rm -rf tmp/doctest
+mkdir -p tmp/doctest
+
+for file in "${files[@]}"; do
+    echo "Extracting doctests from $file..."
+    mkdir tmp/doctest/"$(basename "$file")"
+
+    for start_marker_lineno in $(grep -n '^```python$' "$file" | cut -d: -f1); do
+        outfile="tmp/doctest/$(basename "$file")/$((start_marker_lineno + 1)).jou"
+        awk -v n=$start_marker_lineno '(/^```$/ && line > n) { stop=1 } (++line > n && !stop) { print }' "$file" > "$outfile"
+
+        # Do not test if there is no expected output/errors
+        if [ -z "$(generate_expected_output "$outfile")" ]; then
+            rm "$outfile"
+        fi
+    done
+done
+
+ntotal=0
+nfail=0
+
+cd tmp/doctest
+for file in */*.jou; do
+    echo "${file%.*}" | tr '/' ':'  # foo.md/123.jou --> foo.md:123
+    cp "$file" test.jou
+    if diff --text -u --color=always <(generate_expected_output test.jou | tr -d '\r') <( ("$jou" test.jou 2>&1 || true) | tr -d '\r'); then
+        echo "  ok"
+    else
+        ((nfail++)) || true
+    fi
+    ((ntotal++)) || true
+done
+
+if [ $ntotal == 0 ]; then
+    echo "*** Error: no doctests found ***" >&2
+    exit 1
+fi
+
+echo ""
+echo ""
+
+echo "$((ntotal-nfail)) succeeded, $nfail failed"
+if [ $nfail != 0 ]; then
+    exit 1
+fi