diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8638cba3..4242bdc4 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -97,21 +97,29 @@ jobs:
           tags: |
             permitio/opal-server:test
 
-      # TEST PHASE
-      - name: Create modified docker compose file
-        run: sed 's/:latest/:test/g' docker/docker-compose-with-callbacks.yml > docker/docker-compose-test.yml
-
-      - name: Bring up stack
-        run: docker-compose -f docker/docker-compose-test.yml up -d
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
 
-      - name: Check if OPA is healthy
-        run: ./scripts/wait-for.sh -t 2 http://localhost:8181/v1/data/users -- sleep 10 && curl -s "http://localhost:8181/v1/data/users" | jq '.result.bob.location.country == "US"'
+      - name: Install opal packages
+        run: |
+          python -m pip install -e ./packages/opal-common
+          python -m pip install -e ./packages/opal-client
+          python -m pip install -e ./packages/opal-server
+
+      - name: App Tests
+        working-directory: ./app-tests
+        env:
+          OPAL_TESTS_POLICY_REPO_DEPLOY_KEY: ${{ secrets.OPAL_TESTS_POLICY_REPO_DEPLOY_KEY }}
+        run: |
+          # Prepare git for using tests policy repo
+          export POLICY_REPO_SSH_KEY_PATH=$(realpath ./opal-tests-policy-repo-key)
+          echo "$OPAL_TESTS_POLICY_REPO_DEPLOY_KEY" > $POLICY_REPO_SSH_KEY_PATH
+          chmod 400 $POLICY_REPO_SSH_KEY_PATH
 
-      - name: Output container logs
-        run: docker-compose -f docker/docker-compose-test.yml logs
+          git config --global core.sshCommand "ssh -i $POLICY_REPO_SSH_KEY_PATH -o IdentitiesOnly=yes"
+          git config --global user.name "$GITHUB_ACTOR"
+          git config --global user.email "<>"
 
-      - name: check if opal-client was brought up successfully
-        run: |
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle"
-          docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204'
+          ./run.sh
diff --git a/docker/docker-compose-with-everything.yml b/app-tests/docker-compose-app-tests.yml
similarity index 94%
rename from docker/docker-compose-with-everything.yml
rename to app-tests/docker-compose-app-tests.yml
index 6bdc6aa3..589a642d 100644
--- a/docker/docker-compose-with-everything.yml
+++ b/app-tests/docker-compose-app-tests.yml
@@ -5,8 +5,9 @@ services:
       - POSTGRES_DB=postgres
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
+
   opal_server:
-    image: permitio/opal-server:latest
+    image: permitio/opal-server:test
     deploy:
       mode: replicated
       replicas: 2
@@ -16,7 +17,7 @@ services:
       - UVICORN_NUM_WORKERS=4
       - OPAL_POLICY_REPO_URL=git@github.com:permitio/opal-tests-policy-repo.git
       - OPAL_POLICY_REPO_MAIN_BRANCH=${POLICY_REPO_BRANCH}
-      - OPAL_POLICY_REPO_SSH_KEY=${POLICY_REPO_SSH_KEY}
+      - OPAL_POLICY_REPO_SSH_KEY=${OPAL_POLICY_REPO_SSH_KEY}
       - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}}
       - OPAL_LOG_FORMAT_INCLUDE_PID=true
       - OPAL_POLICY_REPO_WEBHOOK_SECRET=xxxxx
@@ -31,8 +32,9 @@ services:
       - "7002-7003:7002"
     depends_on:
       - broadcast_channel
+
   opal_client:
-    image: permitio/opal-client:latest
+    image: permitio/opal-client:test
     deploy:
       mode: replicated
       replicas: 2
diff --git a/docker/app_tests.sh b/app-tests/run.sh
similarity index 82%
rename from docker/app_tests.sh
rename to app-tests/run.sh
index 13f0fc77..6195c056 100755
--- a/docker/app_tests.sh
+++ b/app-tests/run.sh
@@ -1,12 +1,7 @@
 #!/bin/bash
-set -ex
+set -e
 
-if [ ! -f "docker-compose-with-everything.yml" ]; then
-   echo "did not find compose file - run this script from the 'docker/' directory under opal root!"
-   exit
-fi
-
-# TODO: Pretiffy exports
+# TODO: Prettify exports
 ssh-keygen -q -t rsa -b 4096 -m pem -f opal_crypto_key -N ""
 
 export OPAL_AUTH_PUBLIC_KEY=`cat opal_crypto_key.pub`
@@ -16,18 +11,6 @@ rm opal_crypto_key.pub opal_crypto_key
 echo "generating master token..."
 export OPAL_AUTH_MASTER_TOKEN=`openssl rand -hex 16`
 
-if ! command -v opal-server &> /dev/null
-then
-    echo "opal-server cli was not found, run: 'pip install opal-server'"
-    exit
-fi
-
-if ! command -v opal-client &> /dev/null
-then
-    echo "opal-client cli was not found, run: 'pip install opal-client'"
-    exit
-fi
-
 OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/ OPAL_AUTH_JWT_ISSUER=https://opal.ac/ OPAL_REPO_WATCHER_ENABLED=0 opal-server run &
 
 sleep 2;
@@ -39,6 +22,7 @@ ps -ef | grep opal | grep -v grep | awk '{print $2}' | xargs kill
 
 sleep 5;
 
+echo "create .env file"
 rm -f .env
 echo "OPAL_AUTH_PUBLIC_KEY=\"$OPAL_AUTH_PUBLIC_KEY\"" >> .env
 echo "OPAL_AUTH_PRIVATE_KEY=\"$OPAL_AUTH_PRIVATE_KEY\"" >> .env
@@ -46,8 +30,8 @@ echo "OPAL_AUTH_MASTER_TOKEN=\"$OPAL_AUTH_MASTER_TOKEN\"" >> .env
 echo "OPAL_CLIENT_TOKEN=\"$OPAL_CLIENT_TOKEN\"" >> .env
 echo "OPAL_AUTH_PRIVATE_KEY_PASSPHRASE=\"$OPAL_AUTH_PRIVATE_KEY_PASSPHRASE\"" >> .env
 
-
 # Clone tests repo & create testing branch
+echo "clone opal policy test git repo"
 export POLICY_REPO_BRANCH
 POLICY_REPO_BRANCH=test-$RANDOM$RANDOM
 rm -rf ./opal-tests-policy-repo
@@ -57,14 +41,15 @@ git checkout -b $POLICY_REPO_BRANCH
 git push --set-upstream origin $POLICY_REPO_BRANCH
 cd -
 
-export POLICY_REPO_SSH_KEY
-POLICY_REPO_SSH_KEY=${POLICY_REPO_SSH_KEY:=$(cat ~/.ssh/id_rsa)}
+export OPAL_POLICY_REPO_SSH_KEY
+OPAL_POLICY_REPO_SSH_KEY=$(cat "$POLICY_REPO_SSH_KEY_PATH")
 
 function compose {
-  docker compose -f docker-compose-with-everything.yml --env-file .env "$@"
+  docker compose -f ./docker-compose-app-tests.yml --env-file .env "$@"
 }
 
 function check_clients_logged {
+  echo "- Looking for msg '$1' in client's logs"
   compose logs --index 1 opal_client | grep -q "$1"
   compose logs --index 2 opal_client | grep -q "$1"
 }
@@ -80,10 +65,12 @@ function check_no_error {
 function clean_up {
     ARG=$?
     if [[ "$ARG" -ne 0 ]]; then
-      # compose logs
-      echo "Failed test"
+      echo "*** Test Failed ***"
+      echo ""
+      compose logs
     else
-      echo "Success"
+      echo "*** Test Passed ***"
+      echo ""
     fi
     compose down
     cd opal-tests-policy-repo; git push -d origin $POLICY_REPO_BRANCH; cd - # Remove remote tests branch
@@ -103,6 +90,7 @@ check_clients_logged 'PUT /v1/data/static -> 204'
 check_no_error
 
 function test_push_policy {
+  echo "- Testing pushing policy $1"
   regofile="$1.rego"
   cd opal-tests-policy-repo
   echo "package $1" > "$regofile"
@@ -117,6 +105,7 @@ function test_push_policy {
 }
 
 function test_data_publish {
+  echo "- Testing data publish for user $1"
   user=$1
   OPAL_CLIENT_TOKEN=$OPAL_DATA_SOURCE_TOEN opal-client publish-data-update --src-url https://api.country.is/23.54.6.78 -t policy_data --dst-path "/users/$user/location"
   sleep 5
@@ -128,6 +117,7 @@ test_push_policy "something"
 
 # TODO: Test statistic
 
+echo "- Testing broadcast channel disconnection"
 compose restart broadcast_channel
 sleep 10