tattle-made · dennyabrain · Feb 29, 2024 · Feb 28, 2024 · Feb 28, 2024 · Feb 28, 2024
diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml
@@ -0,0 +1,61 @@
+services:
+  sut:
+    build:
+      context: ./../src
+      dockerfile: Dockerfile.test
+      target: test
+    env_file: ./../src/test.env
+    command: bash -c "python -c 'import time';
+      while ! curl store:9200 | grep -q 'You Know, for Search' 
+      && ! curl queue:15672 | grep -q '<!DOCTYPE html>'; 
+      do python -c 'time.sleep(10)'; 
+      done;
+      nose2 tests"
+    links:
+      - store
+      - queue
+    depends_on:
+      store:
+        condition: service_started
+      queue:
+        condition: service_started
+
+  store:
+    container_name: es
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
+    volumes:
+      - ./.docker/es/data:/var/lib/elasticsearch/data
+#    ports:
+#      - "9300:9300"
+#      - "9200:9200"
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+      - http.cors.enabled=true
+      - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358
+      - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
+      - http.cors.allow-credentials=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK
+
+  queue:
+    image: rabbitmq:3.12.12-management
+    container_name: rabbitmq
+    hostname: rabbit
+    volumes:
+      - ./.docker/rabbitmq/data:/var/lib/rabbitmq
+      - ./.docker/rabbitmq/logs:/var/log/rabbitmq
+    environment:
+      RABBITMQ_ERLANG_COOKIE: "secret-cookie"
+      RABBITMQ_DEFAULT_USER: "admin"
+      RABBITMQ_DEFAULT_PASS: "Admin123"
+#    ports:
+#      - "5672:5672"
+#      - "15672:15672"
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
@@ -0,0 +1,38 @@
+name: Run tests on PR
+
+on:
+  pull_request:
+    branches:
+      - development
+      - hotfix
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+
+jobs:
+  test:
+    if: github.event.pull_request.draft == false
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build container for tests
+        run: docker compose -f ./.ci/docker-compose.yml -p ci build
+
+      - name: Launch containers to run tests
+        id: vars
+        run: |
+          docker compose -f ./.ci/docker-compose.yml -p ci up -d store queue sut
+          docker logs -f ci-sut-1
+          echo "sut_output=$(docker wait ci-sut-1)" >> $GITHUB_OUTPUT
+
+      - name: Check failure
+        run: |
+          if [ ${{ steps.vars.outputs.sut_output }} != 0 ]; then
+            echo ${{ steps.vars.outputs.sut_output }}
+            exit 1
+          fi
diff --git a/.gitignore b/.gitignore
@@ -93,6 +93,7 @@ celerybeat-schedule
 .env
 .env.local
 *.env
+!test.env
 
 # virtualenv
 .venv/
@@ -120,6 +121,7 @@ credentials.json
 .DS_Store
 
 # Docker logs
+.ci/.docker/*
 .docker/*
 # src/core/operators/sample_data/
 # sample_data/**

diff --git a/src/Dockerfile.test b/src/Dockerfile.test
@@ -0,0 +1,18 @@
+FROM python:3.11-slim-bullseye AS base
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && rm -rf /var/lib/apt/lists/*
+ENV PATH=/root/.local/bin:$PATH
+RUN pip install --no-cache-dir --upgrade pip
+RUN apt-get update && apt-get -y upgrade && apt-get install -y wget curl grep
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir --user -r requirements.txt
+COPY . /app
+
+#### TEST IMAGE ####
+FROM base AS test
+RUN cd core/operators \
+    && pip install --no-cache-dir --user -r vid_vec_rep_resnet_requirements.txt \
+    && pip install --no-cache-dir --user -r audio_vec_embedding_requirements.txt
+
diff --git a/src/config-server.yml b/src/config-server.yml
@@ -38,9 +38,9 @@ operators :
 #    - name : "Text Extraction"
 #      type : "detect_text_in_image"
 #      parameters: {}
-    - name : "Image Vector Representation"
-      type : "image_vec_rep_resnet"
-      parameters: {}
+#    - name : "Image Vector Representation"
+#      type : "image_vec_rep_resnet"
+#      parameters: {}
     - name : "Video Vector Representation"
       type : "vid_vec_rep_resnet"
       parameters: {

diff --git a/src/requirements.in b/src/requirements.in
@@ -13,4 +13,5 @@ memray==1.11.0  # dev
 pyinstrument==4.6.2
 numpy==1.26.3
 requests==2.31.0
-locust==2.23.1
+locust==2.23.1
+nose2==0.14.1
diff --git a/src/requirements.txt b/src/requirements.txt
@@ -78,6 +78,8 @@ memray==1.11.0
     # via -r requirements.in
 msgpack==1.0.7
     # via locust
+nose2==0.14.1
+    # via -r requirements.in
 numpy==1.26.3
     # via -r requirements.in
 packaging==23.2

diff --git a/src/test.env b/src/test.env
@@ -0,0 +1,7 @@
+MQ_USERNAME=admin
+MQ_PASSWORD=Admin123
+MQ_HOST=rabbitmq
+ES_HOST=es
+ES_IMG_INDEX=imgsearch
+ES_TXT_INDEX=txtsearch
+ES_VID_INDEX=vidsearch
diff --git a/src/tests/test_audio_es_vec.py b/src/tests/test_audio_es_vec.py
@@ -81,7 +81,7 @@ def test_store_audio(self):
         # print(result)
         self.assertEqual(result["result"], "created")
 
-    # @skip
+    @skip
     def test_store_and_search_audio(self):
         es = ES(self.param)
         es.connect()

diff --git a/src/tests/test_index_api_as_client.py b/src/tests/test_index_api_as_client.py
@@ -34,7 +34,7 @@ def testIndexText(self):
             self.assertEqual(response.status_code, 200)
         # self.assertEqual(len(response.json()["vector_representation"]), 768)
 
-    # @skip
+    @skip
     def testIndexImage(self):
         url = API_URL + "/index"
         headers = {"Content-Type": "application/json"}
@@ -60,7 +60,7 @@ def testIndexImage(self):
         print(response.text)
         self.assertEqual(response.status_code, 200)
 
-    # @skip
+    @skip
     def testIndexVideo(self):
         url = API_URL + "/index"
         headers = {"Content-Type": "application/json"}

diff --git a/src/tests/test_search_api_as_client.py b/src/tests/test_search_api_as_client.py
@@ -24,7 +24,7 @@ def testSearchRawQuery(self):
         # print(response.json())
         self.assertEqual(response.status_code, 200)
 
-    # @skip
+    @skip
     def testSearchImage(self):
         url = API_URL + "/search"
         data = {"query_type": "image"}
@@ -36,7 +36,7 @@ def testSearchImage(self):
             self.assertEqual(response.status_code, 200)
 
 
-    # @skip
+    @skip
     def testIndexVideo(self):
         url = API_URL + "/search"
         with open("tests/sample_data/sample-cat-video.mp4", "rb") as file: