Improve testing

- `make test` now works whether you are inside or
  outside the docker container
- tests now better handle errors that cause the
  `actual` psql variable to be unset
- fail if no tests were run

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.3-labs
 FROM postgres:16
-
+ENV WHERE_AM_I=docker
 ENV DEBIAN_FRONTEND=noninteractive
 USER root
 

test.sh

@@ -8,35 +8,12 @@ if [ -f .env ]; then
   set +a
 fi
 
-if [ -z "$ENABLE_OPENAI_TESTS" ]; then
-  export ENABLE_OPENAI_TESTS=0
+if [ -n "$WHERE_AM_I" ] && [ "$WHERE_AM_I" == "docker" ]; then
+  if [ "$(whoami)" == "root" ]; then
+    echo switching to postgres user...
+    su postgres -
+  fi
+  psql -d postgres -f test.sql
+else
+  psql --no-psqlrc -d 'postgres://[email protected]:9876/postgres' -f test.sql
 fi
-
-if [ "$ENABLE_OPENAI_TESTS" ] && [ -z "$OPENAI_API_KEY" ]; then
-  echo "OPENAI_API_KEY must be set if running OpenAI tests"
-  exit 3
-fi
-
-if [ -z "$ENABLE_OLLAMA_TESTS" ]; then
-  export ENABLE_OLLAMA_TESTS=0
-fi
-
-if [ -z "$ENABLE_ANTHROPIC_TESTS" ]; then
-  export ENABLE_ANTHROPIC_TESTS=0
-fi
-
-if [ "$ENABLE_ANTHROPIC_TESTS" ] && [ -z "$ANTHROPIC_API_KEY" ]; then
-  echo "ANTHROPIC_API_KEY must be set if running Anthropic tests"
-  exit 3
-fi
-
-if [ -z "$ENABLE_COHERE_TESTS" ]; then
-  export ENABLE_COHERE_TESTS=0
-fi
-
-if [ "$ENABLE_COHERE_TESTS" ] && [ -z "$COHERE_API_KEY" ]; then
-  echo "COHERE_API_KEY must be set if running Cohere tests"
-  exit 3
-fi
-
-psql -d postgres -f test.sql

test.sql

@@ -83,7 +83,7 @@ create table tests
 );
 
 -------------------------------------------------------------------------------
--- convenience functions for recording test results
+-- convenience function for recording test results
 create function result(_test text, _expected text, _actual text) returns bool
 as $func$
 merge into tests as t
@@ -95,92 +95,121 @@ when not matched then insert (test, actual) values (x.test, x.actual)
 select passed from tests where test = _test;
 $func$ language sql;
 
-create function result(_test text, _expected int, _actual int) returns bool
-return (select result(_test, _expected::text, _actual::text))
-;
-
-create function result(_test text, _expected bool, _actual bool) returns bool
-return (select result(_test, _expected::text, _actual::text))
-;
-
 \pset tuples_only on
 -------------------------------------------------------------------------------
 -- openai tests
 \getenv enable_openai_tests ENABLE_OPENAI_TESTS
+\if :{?enable_openai_tests}
+\else
+\set enable_openai_tests 0
+\endif
 \if :enable_openai_tests
 \set ON_ERROR_ROLLBACK on
 \set ON_ERROR_STOP off
 \i tests/openai.sql
 \set ON_ERROR_ROLLBACK off
 \set ON_ERROR_STOP on
+\else
+\echo Skipped OpenAI tests
 \endif
 
 -------------------------------------------------------------------------------
 -- ollama tests
 \getenv enable_ollama_tests ENABLE_OLLAMA_TESTS
+\if :{?enable_ollama_tests}
+\else
+\set enable_ollama_tests 0
+\endif
 \if :enable_ollama_tests
 \set ON_ERROR_ROLLBACK on
 \set ON_ERROR_STOP off
 \i tests/ollama.sql
 \set ON_ERROR_ROLLBACK off
 \set ON_ERROR_STOP on
+\else
+\echo Skipped Ollama tests
 \endif
 
 -------------------------------------------------------------------------------
 -- anthropic tests
 \getenv enable_anthropic_tests ENABLE_ANTHROPIC_TESTS
+\if :{?enable_anthropic_tests}
+\else
+\set enable_anthropic_tests 0
+\endif
 \if :enable_anthropic_tests
 \set ON_ERROR_ROLLBACK on
 \set ON_ERROR_STOP off
 \i tests/anthropic.sql
 \set ON_ERROR_ROLLBACK off
 \set ON_ERROR_STOP on
+\else
+\echo Skipped Anthropic tests
 \endif
 
 -------------------------------------------------------------------------------
 -- cohere tests
 \getenv enable_cohere_tests ENABLE_COHERE_TESTS
+\if :{?enable_cohere_tests}
+\else
+\set enable_cohere_tests 0
+\endif
 \if :enable_cohere_tests
 \set ON_ERROR_ROLLBACK on
 \set ON_ERROR_STOP off
 \i tests/cohere.sql
 \set ON_ERROR_ROLLBACK off
 \set ON_ERROR_STOP on
+\else
+\echo Skipped Cohere tests
 \endif
 
 \pset tuples_only off
 -------------------------------------------------------------------------------
 -- test results
 \echo
 \echo
-\echo test results
+\echo
 \echo
 \echo
 
 \set ON_ERROR_STOP on
 \set ON_ERROR_ROLLBACK off
-\echo test results
-select test, passed
-from tests
-;
 
-\echo failed tests
-select *
+-- we should fail if no tests were run
+select count(*) > 0 as result
 from tests
-where passed is distinct from true
-;
+\gset
 
-\echo test stats
-select
-  count(*) as total
-, count(*) filter (where passed = true) as passed
-, count(*) filter (where passed is distinct from true) as failed
-from tests
-;
+\if :result
+
+    \echo test results
+    select test, passed
+    from tests
+    ;
+
+    \echo failed tests
+    select *
+    from tests
+    where passed is distinct from true
+    ;
+
+    \echo test stats
+    select
+      count(*) as total
+    , count(*) filter (where passed = true) as passed
+    , count(*) filter (where passed is distinct from true) as failed
+    from tests
+    ;
+
+    select count(*) filter (where passed is distinct from true) = 0 as result
+    from tests
+    \gset
+
+\else
+\warn NO TESTS WERE RUN!
+\endif
 
-select count(*) filter (where passed is distinct from true) = 0 as result
-from tests
-\gset
 
 reset role; -- no longer tester
 

tests/anthropic.sql

@@ -2,6 +2,17 @@
 -- get our anthropic api key
 
 \getenv anthropic_api_key ANTHROPIC_API_KEY
+\if :{?anthropic_api_key}
+\else
+\warn Anthropic tests are enabled but ANTHROPIC_API_KEY is not set!
+do $$
+begin
+raise exception 'Anthropic tests are enabled but ANTHROPIC_API_KEY is not set!';
+end;
+$$;
+\q
+\endif
+
 -- set our session local GUC
 select set_config('ai.anthropic_api_key', $1, false) is not null as set_anthropic_api_key
 \bind :anthropic_api_key
@@ -18,7 +29,9 @@ values
 
 -------------------------------------------------------------------------------
 -- anthropic_generate
-\echo anthropic_generate
+\set testname anthropic_generate
+\set expected t
+\echo :testname
 select anthropic_generate
 ( 'claude-3-5-sonnet-20240620'
 , jsonb_build_array
@@ -32,15 +45,18 @@ select anthropic_generate
 \bind :anthropic_api_key
 \gset
 
+\if :{?actual}
 select jsonb_extract_path_text(:'actual'::jsonb, 'content', '0', 'text') is not null and (:'actual'::jsonb)->>'stop_reason' = 'end_turn' as actual
 \gset
+\endif
 
-select result('anthropic_generate', true, :'actual');
-\unset actual
+\ir eval.sql
 
 -------------------------------------------------------------------------------
 -- anthropic_generate-no-key
-\echo anthropic_generate-no-key
+\set testname anthropic_generate-no-key
+\set expected t
+\echo :testname
 select anthropic_generate
 ( 'claude-3-5-sonnet-20240620'
 , jsonb_build_array
@@ -52,8 +68,9 @@ select anthropic_generate
 ) as actual
 \gset
 
+\if :{?actual}
 select jsonb_extract_path_text(:'actual'::jsonb, 'content', '0', 'text') is not null and (:'actual'::jsonb)->>'stop_reason' = 'end_turn' as actual
 \gset
+\endif
 
-select result('anthropic_generate-no-key', true, :'actual');
-\unset actual
+\ir eval.sql