feat: Add 15m timeout for CI benchmarks

qdrant · Jun 5, 2024 · 96da4d8 · 96da4d8
1 parent 884fda3
commit 96da4d8
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 2 deletions.
diff --git a/.github/workflows/continuous-benchmark.yaml b/.github/workflows/continuous-benchmark.yaml
@@ -16,6 +16,7 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - name: Benches
+        id: benches
         run: |
             export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
             export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
@@ -28,15 +29,22 @@ jobs:
             DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark"
             DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark"
 
+            set +e
+
             for dataset in "${!DATASET_TO_ENGINE[@]}"; do
               export ENGINE_NAME=${DATASET_TO_ENGINE[$dataset]}
               export DATASETS=$dataset
 
               # Benchmark the dev branch:
               export QDRANT_VERSION=ghcr/dev
-              bash -x tools/run_ci.sh
+              timeout 15m bash -x tools/run_ci.sh
 
               # Benchmark the master branch:
               export QDRANT_VERSION=docker/master
-              bash -x tools/run_ci.sh
+              timeout 15m bash -x tools/run_ci.sh
             done
+
+            set -e
+      - name: Fail job if any of the benches failed
+        if: steps.benches.outputs.failed == 'true'
+        run: exit 1
diff --git a/tools/run_ci.sh b/tools/run_ci.sh
@@ -2,6 +2,19 @@
 
 set -e
 
+function handle_error() {
+  echo "Error occured ${QDRANT_VERSION[@]} ${ENGINE_NAME[@]} ${DATASETS[@]}"
+  echo "::set-output name=failed::true"
+}
+
+function handle_term() {
+  echo "Timeout occured ${QDRANT_VERSION[@]} ${ENGINE_NAME[@]} ${DATASETS[@]}"
+  echo "::set-output name=failed::true"
+}
+
+trap handle_err ERR
+trap handle_term TERM
+
 # Script, that runs benchmark within the GitHub Actions CI environment
 
 SCRIPT=$(realpath "$0")