Fix failure exit code regression

dctest should exit 1 if any test fails. We should be checking this
instead of swallowing all exit codes in GHA.

.github/workflows/push.yaml

@@ -24,9 +24,22 @@ jobs:
       - name: Run Tests
         run: ./test/runtests
 
+  test-examples-node:
+    name: Test Examples (Node)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with: { submodules: 'recursive', fetch-depth: 0 }
+
+      - name: Install NPM
+        run: npm install
+
+      - name: Run Examples
+        run: ./test/runexamples
 
   test-examples:
-    name: Test Examples
+    name: Test Examples (Docker)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -37,51 +50,10 @@ jobs:
         run: |
           docker build -t dctest .
 
-      - name: Setup Fixtures
-        run: |
-          docker compose -f examples/docker-compose.yaml up -d
-
       - name: Install NPM (for schema validation)
         run: npm install
 
-      - name: Run Intro Examples
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-      - name: Run Intro + Fail Examples with --continue-on-error
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-      - name: Run Dependency Examples
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-
-      - name: Setup Fixtures
-        if: always()
-        run: |
-          docker compose -f examples/docker-compose.yaml down -t 1 --remove-orphans
+      - name: Run Examples
+        env:
+          DCTEST_IMAGE: dctest
+        run: ./test/runexamples

src/dctest/core.cljs

@@ -7,7 +7,7 @@
             [clojure.string :as S]
             [clojure.walk :refer [postwalk]]
             [dctest.expressions :as expr]
-            [dctest.outcome :refer [failure? pending? pending-> short-outcome
+            [dctest.outcome :refer [failure? passed? pending? pending-> short-outcome
                                     fail! pass! skip!]]
             [dctest.util :as util :refer [obj->str js->map log indent indent-print-table-str]]
             [promesa.core :as P]
@@ -487,4 +487,4 @@ Options:
     (write-results-file opts summary)
     (print-results opts summary)
 
-    (when (failure? summary) (fatal 1))))
+    (when-not (passed? (:summary summary)) (fatal 1))))

src/dctest/outcome.cljs

@@ -6,6 +6,9 @@
 (defn failure? [{:keys [outcome]}]
   (= :failed outcome))
 
+(defn passed? [{:keys [outcome]}]
+  (= :passed outcome))
+
 (defn pending? [{:keys [outcome]}]
   (= :pending outcome))
 

test/runexamples

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+PROJECT=${PROJECT:-${USER}}
+DCTEST_IMAGE=${DCTEST_IMAGE}
+
+repo_root=$(dirname $(readlink -f "${0}"))/..
+examples_dir=${repo_root}/examples
+
+results_dir=$(mktemp -d)
+
+dc() { docker compose -p ${PROJECT} -f ${examples_dir}/docker-compose.yaml "${@}"; }
+up() { dc up -d; }
+down() { dc down; }
+
+fail() {
+  msg=${1}
+  down
+  echo $msg
+  exit 1
+}
+
+expect() {
+  passed=${1}; shift
+  failed=${1}; shift
+  example_files=${@}
+
+  if [ -z "${DCTEST_IMAGE}" ]; then
+    cmd="${repo_root}/dctest --continue-on-error --results-file ${results_dir}/results.json ${PROJECT}"
+    for example in ${example_files}; do
+      cmd="${cmd} ${examples_dir}/${example}"
+    done
+  else
+    cmd="docker run --rm -v /var/run/docker.sock:/var/run/docker.sock"
+    cmd="${cmd} -v ${repo_root}/examples/:/app/examples -v ${results_dir}/:/app/results"
+    cmd="${cmd} ${DCTEST_IMAGE} --continue-on-error --results-file /app/results/results.json ${PROJECT}"
+    for example in ${example_files}; do
+      cmd="${cmd} /app/examples/${example}"
+    done
+  fi
+
+  echo "Running: ${cmd}"
+  ${cmd}
+  exitcode=$?
+
+  if [ "${failed}" == "0" ]; then
+    expected_exitcode="0"
+  else
+    expected_exitcode="1"
+  fi
+
+  [ "${expected_exitcode}" == "${exitcode}" ] \
+    || fail "Exit code ${exitcode} doesn't match expected ${expected_exitcode}"
+
+  jq --exit-status '[.summary.passed == '${passed}', .summary.failed == '${failed}'] | all' ${results_dir}/results.json \
+    || fail "Results file does not match expected passed/fail rate"
+
+  node ${repo_root}/scripts/validateSchema.js ${results_dir}/results.json ${repo_root}/schemas/results-file.yaml \
+    || fail "Results file does not match schema"
+}
+
+
+# TESTS
+
+up
+
+expect 5  0 00-intro.yaml
+expect 6  4 00-intro.yaml 01-fails.yaml
+expect 12 0 02-deps.yaml
+
+down