tattle-made · duggalsu · Mar 5, 2024 · Feb 28, 2024 · Feb 28, 2024 · Feb 28, 2024
diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml
@@ -0,0 +1,61 @@
+services:
+  sut:
+    build:
+      context: ./../src
+      dockerfile: Dockerfile.test
+      target: test
+    env_file: ./../src/test.env
+    command: bash -c "python -c 'import time';
+      while ! curl store:9200 | grep -q 'You Know, for Search' 
+      && ! curl queue:15672 | grep -q '<!DOCTYPE html>'; 
+      do python -c 'time.sleep(10)'; 
+      done;
+      nose2 tests"
+    links:
+      - store
+      - queue
+    depends_on:
+      store:
+        condition: service_started
+      queue:
+        condition: service_started
+
+  store:
+    container_name: es
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
+    volumes:
+      - ./.docker/es/data:/var/lib/elasticsearch/data
+#    ports:
+#      - "9300:9300"
+#      - "9200:9200"
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+      - http.cors.enabled=true
+      - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358
+      - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
+      - http.cors.allow-credentials=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK
+
+  queue:
+    image: rabbitmq:3.12.12-management
+    container_name: rabbitmq
+    hostname: rabbit
+    volumes:
+      - ./.docker/rabbitmq/data:/var/lib/rabbitmq
+      - ./.docker/rabbitmq/logs:/var/log/rabbitmq
+    environment:
+      RABBITMQ_ERLANG_COOKIE: "secret-cookie"
+      RABBITMQ_DEFAULT_USER: "admin"
+      RABBITMQ_DEFAULT_PASS: "Admin123"
+#    ports:
+#      - "5672:5672"
+#      - "15672:15672"
diff --git a/.github/workflows/docker-push-audiovec-worker-staging.yml b/.github/workflows/docker-push-audiovec-worker-staging.yml
@@ -0,0 +1,33 @@
+name: Publish Audio Worker to Dockerhub for Staging
+
+on: workflow_dispatch
+
+jobs:
+  api:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: development
+      - name: "Setup Node version"
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.9.0
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: |
+          echo "setting variables"
+          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - uses: elgohr/Publish-Docker-Github-Action@main
+        with:
+          username: tattletech
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: tattletech/feluda-operator-audiovec
+          workdir: src/
+          dockerfile: worker/audiovec/Dockerfile.audio_worker
+          tags: worker-amd64-latest
+          platforms: linux/amd64
diff --git a/.github/workflows/docker-push-vidvec.yml → ...orkflows/docker-push-vidvec-benchmark.yml b/.github/workflows/docker-push-vidvec.yml → ...orkflows/docker-push-vidvec-benchmark.yml
@@ -27,18 +27,18 @@ jobs:
           username: tattletech
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-vidvec
-          workdir: src/benchmark/vidvec/
-          dockerfile: Dockerfile.vid_vec_rep_resnet
-          tags: ${{ steps.vars.outputs.sha_short }}-amd64
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
+          tags: benchmark-amd64-latest
           platforms: linux/amd64
       - uses: elgohr/Publish-Docker-Github-Action@main
         with:
           username: tattletech
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-vidvec
-          workdir: src/benchmark/vidvec/
-          dockerfile: Dockerfile.vid_vec_rep_resnet.graviton
-          tags: ${{ steps.vars.outputs.sha_short }}-arm64
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton
+          tags: benchmark-arm64-latest
           platforms: linux/arm64
 
       # - name: deploy to cluster

diff --git a/.github/workflows/docker-push-vidvec-worker-staging.yml b/.github/workflows/docker-push-vidvec-worker-staging.yml
@@ -27,7 +27,7 @@ jobs:
           username: tattletech
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-vidvec
-          workdir: src/worker/vidvec/
-          dockerfile: Dockerfile.test_video_worker
-          tags: latest-amd64
+          workdir: src/
+          dockerfile: worker/vidvec/Dockerfile.video_worker
+          tags: worker-amd64-latest
           platforms: linux/amd64
diff --git a/.github/workflows/merge-dev-hotfix.yml b/.github/workflows/merge-dev-hotfix.yml
@@ -0,0 +1,77 @@
+name: Merge PR to development or hotfix
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - development
+      - hotfix
+
+jobs:
+  if_merged:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    environment:
+      name: staging
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: |
+          echo "setting variables"
+          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+          echo "version=latest" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Publish vidvec amd64 benchmark to dockerhub
+        uses: elgohr/Publish-Docker-Github-Action@main
+        with:
+          username: tattletech
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: tattletech/feluda-operator-vidvec
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
+          tags: benchmark-amd64-${{ steps.vars.outputs.branch }}-${{ steps.vars.outputs.version }}
+          platforms: linux/amd64
+
+      - name: Publish vidvec arm64 benchmark to dockerhub
+        uses: elgohr/Publish-Docker-Github-Action@main
+        with:
+          username: tattletech
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: tattletech/feluda-operator-vidvec
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton
+          tags: benchmark-arm64-${{ steps.vars.outputs.branch }}-${{ steps.vars.outputs.version }}
+          platforms: linux/arm64
+
+#      - name: Publish vidvec amd64 worker to dockerhub
+#        uses: elgohr/Publish-Docker-Github-Action@main
+#        with:
+#          username: tattletech
+#          password: ${{ secrets.DOCKER_PASSWORD }}
+#          name: tattletech/feluda-operator-vidvec
+#          workdir: src/
+#          dockerfile: worker/vidvec/Dockerfile.video_worker
+#          tags: worker-amd64-${{ steps.vars.outputs.branch }}-${{ steps.vars.outputs.version }}
+#          platforms: linux/amd64
+
+#      - name: deploy to cluster
+#        uses: steebchen/[email protected]
+#        with: # defaults to latest kubectl binary version
+#          config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
+#          command: set image --record deployment/feluda-server feluda-server=tattletech/feluda-server:${{ steps.vars.outputs.sha_short }}
+#      - name: verify deployment
+#        uses: steebchen/[email protected]
+#        with:
+#          config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
+#          version: v1.21.0 # specify kubectl binary version explicitly
+#          command: rollout status deployment/feluda-server
diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml
@@ -0,0 +1,98 @@
+name: Merge PR to main
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  if_merged:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    concurrency: release
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python version
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Dry Run to get next release version
+        id: next_version
+        run: |
+          pip install python-semantic-release
+          export NEXT_TAG_VERSION=$(semantic-release --noop version --print)
+          echo "new_tag_version=${NEXT_TAG_VERSION}" >> $GITHUB_OUTPUT
+
+      - name: Python Semantic Release
+        id: release
+        uses: python-semantic-release/python-semantic-release@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+#      - name: Declare some variables
+#        id: vars
+#        shell: bash
+#        run: |
+#          echo "setting variables"
+#          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Publish vidvec amd64 benchmark to dockerhub
+        if: steps.release.outcome == 'success'
+        uses: elgohr/Publish-Docker-Github-Action@main
+        with:
+          username: tattletech
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: tattletech/feluda-operator-vidvec
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet
+          tags: benchmark-amd64-${{ steps.next_version.outputs.new_tag_version }}
+          platforms: linux/amd64
+
+      - name: Publish vidvec arm64 benchmark to dockerhub
+        if: steps.release.outcome == 'success'
+        uses: elgohr/Publish-Docker-Github-Action@main
+        with:
+          username: tattletech
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          name: tattletech/feluda-operator-vidvec
+          workdir: src/
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton
+          tags: benchmark-arm64-${{ steps.next_version.outputs.new_tag_version }}
+          platforms: linux/arm64
+
+#      - name: Publish vidvec amd64 worker to dockerhub
+#        if: steps.release.outcome == 'success'
+#        uses: elgohr/Publish-Docker-Github-Action@main
+#        with:
+#          username: tattletech
+#          password: ${{ secrets.DOCKER_PASSWORD }}
+#          name: tattletech/feluda-operator-vidvec
+#          workdir: src/
+#          dockerfile: worker/vidvec/Dockerfile.video_worker
+#          tags: worker-amd64-${{ steps.next_version.outputs.new_tag_version }}
+#          platforms: linux/amd64
+
+#      - name: deploy to cluster
+#        uses: steebchen/[email protected]
+#        with: # defaults to latest kubectl binary version
+#          config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
+#          command: set image --record deployment/feluda-server feluda-server=tattletech/feluda-server:${{ steps.vars.outputs.sha_short }}
+#      - name: verify deployment
+#        uses: steebchen/[email protected]
+#        with:
+#          config: ${{ secrets.PROD_KUBE_CONFIG_DATA }}
+#          version: v1.21.0 # specify kubectl binary version explicitly
+#          command: rollout status deployment/feluda-server
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
@@ -0,0 +1,38 @@
+name: Run tests on PR
+
+on:
+  pull_request:
+    branches:
+      - development
+      - hotfix
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+
+jobs:
+  test:
+    if: github.event.pull_request.draft == false
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build container for tests
+        run: docker compose -f ./.ci/docker-compose.yml -p ci build
+
+      - name: Launch containers to run tests
+        id: vars
+        run: |
+          docker compose -f ./.ci/docker-compose.yml -p ci up -d store queue sut
+          docker logs -f ci-sut-1
+          echo "sut_output=$(docker wait ci-sut-1)" >> $GITHUB_OUTPUT
+
+      - name: Check failure
+        run: |
+          if [ ${{ steps.vars.outputs.sut_output }} != 0 ]; then
+            echo ${{ steps.vars.outputs.sut_output }}
+            exit 1
+          fi
diff --git a/.gitignore b/.gitignore
@@ -93,6 +93,7 @@ celerybeat-schedule
 .env
 .env.local
 *.env
+!test.env
 
 # virtualenv
 .venv/
@@ -120,6 +121,7 @@ credentials.json
 .DS_Store
 
 # Docker logs
+.ci/.docker/*
 .docker/*
 # src/core/operators/sample_data/
 # sample_data/**