diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ad77e1c..e7c57d2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,9 +26,4 @@ jobs:
         run: cargo build
       - name: Smoke Test
         shell: bash
-        run: |
-          export JGRAB_HOME="$(pwd)/.jgrab"
-          mkdir $JGRAB_HOME
-          cp ./jgrab-runner/build/libs/jgrab.jar $JGRAB_HOME/jgrab.jar
-          ls -al $JGRAB_HOME
-          ./jgrab-client/target/debug/jgrab-client -e "2 + 2"
+        run: jgrab-client/test.sh
diff --git a/jgrab-client/test.sh b/jgrab-client/test.sh
new file mode 100755
index 0000000..9575019
--- /dev/null
+++ b/jgrab-client/test.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR=$(dirname "$0")
+export JGRAB_HOME="$DIR/.jgrab"
+mkdir "$JGRAB_HOME"
+cp "$DIR"/../jgrab-runner/build/libs/jgrab.jar "$JGRAB_HOME"/jgrab.jar
+ls -al "$JGRAB_HOME"
+jgrab="$DIR"/target/debug/jgrab-client
+
+# start the daemon
+$jgrab -t
+
+result=$($jgrab -e "2 + 2")
+
+if test "$result" = "4"
+then
+  echo "OK"
+else
+  echo "Failed, result is not 4: $result"
+  exit 1
+fi
+
+# cleanup
+$jgrab -s
+
+# on Windows, the Java process may not die quickly enough
+sleep 1
+
+rm -r "$JGRAB_HOME"