Merge pull request #58 from duggalsu/docker_opt

Create and optimize Dockerfiles

README.md

@@ -69,8 +69,17 @@ Please create a new Discussion [here](https://github.com/tattle-made/tattle-api/
   $ cd src/api/core/operators/
   $ pip install -r image_vec_rep_resnet_requirements.txt
   $ pip install -r vid_vec_rep_resnet_requirements.txt
+..
+# Create the docker containers
+  $ cd src/api/
+  $ docker build -t image-operator -f Dockerfile.image_vec_rep_resnet .
+  $ docker build -t video-operator -f Dockerfile.vid_vec_rep_resnet .
+# Run the docker image
+  $ docker run image-operator
+  $ docker run video-operator
   ```
 
+
 6. Then, in a new terminal, start the server with:
 
   ```

src/api/Dockerfile

@@ -12,16 +12,15 @@ RUN apt-get update \
     libgl1-mesa-glx libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/* 
 ENV PATH=/root/.local/bin:$PATH
-RUN pip install --upgrade pip
-RUN pip install pip-tools
-RUN apt-get update && apt-get -y upgrade && apt-get install -y vim curl
-RUN apt-get install -y ffmpeg
-RUN apt-get update && \
-    apt-get -y upgrade && \
-    apt-get install -y tesseract-ocr tesseract-ocr-hin
+RUN pip install --no-cache-dir --upgrade pip
+# RUN apt-get update && apt-get -y upgrade && apt-get install -y vim curl
+# RUN apt-get install -y ffmpeg
+# RUN apt-get update && \
+#     apt-get -y upgrade && \
+#     apt-get install -y tesseract-ocr tesseract-ocr-hin
 WORKDIR /app
 COPY requirements.txt /app/requirements.txt
-RUN pip install --user -r requirements.txt
+RUN pip install --no-cache-dir --user -r requirements.txt
 COPY . /app
 EXPOSE 7000
 
@@ -35,7 +34,7 @@ EXPOSE 7000
 #### DEBUG IMAGE ####
 FROM base AS debug
 RUN apt-get update && apt-get install -y vim zsh jq
-RUN pip install debugpy nose2
+RUN pip install --no-cache-dir debugpy nose2
 RUN export FLASK_DEBUG=1
 CMD python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m flask run -h 0.0.0.0 -p 5000
 

src/api/Dockerfile.image_vec_rep_resnet

@@ -0,0 +1,17 @@
+FROM python:3.11-slim@sha256:637774748f62b832dc11e7b286e48cd716727ed04b45a0322776c01bc526afc3 AS base
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get install -y \
+    --no-install-recommends gcc build-essential \
+    --no-install-recommends libgl1-mesa-glx libglib2.0-0 \
+    && apt-get purge -y --auto-remove \
+    gcc build-essential \
+    libgl1-mesa-glx libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/* 
+ENV PATH=/root/.local/bin:$PATH
+RUN pip install --no-cache-dir --upgrade pip
+WORKDIR /app
+COPY ./core/operators/image_vec_rep_resnet_requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir --user -r requirements.txt
+COPY . /app
+CMD tail -f /dev/null

src/api/Dockerfile.vid_vec_rep_resnet

@@ -0,0 +1,17 @@
+FROM python:3.11-slim@sha256:637774748f62b832dc11e7b286e48cd716727ed04b45a0322776c01bc526afc3 AS base
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get install -y \
+    --no-install-recommends gcc build-essential \
+    --no-install-recommends libgl1-mesa-glx libglib2.0-0 \
+    && apt-get purge -y --auto-remove \
+    gcc build-essential \
+    libgl1-mesa-glx libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/* 
+ENV PATH=/root/.local/bin:$PATH
+RUN pip install --no-cache-dir --upgrade pip
+WORKDIR /app
+COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir --user -r requirements.txt
+COPY . /app
+CMD tail -f /dev/null

src/api/core/operators/detect_text_in_image_requirements.in

@@ -1,3 +1,3 @@
 google-cloud==0.34.0
 google-cloud-vision==3.5.0
-boto3==1.34.19
+boto3==1.34.34

src/api/core/operators/detect_text_in_image_requirements.txt

@@ -4,9 +4,9 @@
 #
 #    pip-compile detect_text_in_image_requirements.in
 #
-boto3==1.34.19
+boto3==1.34.34
     # via -r detect_text_in_image_requirements.in
-botocore==1.34.25
+botocore==1.34.35
     # via
     #   boto3
     #   s3transfer

src/api/core/operators/image_vec_rep_resnet.py

@@ -13,7 +13,7 @@
 class ResNet18:
     def __init__(self):
         print("Initializing ResNet")
-        self.model = models.resnet18(pretrained=True)
+        self.model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
         self.feature_layer = self.model._modules.get("avgpool")
         self.model.eval()
 

src/api/core/operators/image_vec_rep_resnet_requirements.in

@@ -1,3 +1,4 @@
-pytest==7.4.4
-nltk==3.8.1
-textblob==0.17.1
+torch==2.1.2+cpu
+torchvision==0.16.2+cpu
+numpy==1.26.3
+Pillow==10.2.0  # dev

src/api/core/operators/image_vec_rep_resnet_requirements.txt

@@ -2,27 +2,47 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile image_vec_rep_resnet_requirements.in
+#    pip-compile --find-links=https://download.pytorch.org/whl/torch_stable.html image_vec_rep_resnet_requirements.in
 #
-click==8.1.7
-    # via nltk
-iniconfig==2.0.0
-    # via pytest
-joblib==1.3.2
-    # via nltk
-nltk==3.8.1
+--find-links https://download.pytorch.org/whl/torch_stable.html
+
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+filelock==3.13.1
+    # via torch
+fsspec==2024.2.0
+    # via torch
+idna==3.6
+    # via requests
+jinja2==3.1.3
+    # via torch
+markupsafe==2.1.5
+    # via jinja2
+mpmath==1.3.0
+    # via sympy
+networkx==3.2.1
+    # via torch
+numpy==1.26.3
     # via
     #   -r image_vec_rep_resnet_requirements.in
-    #   textblob
-packaging==23.2
-    # via pytest
-pluggy==1.3.0
-    # via pytest
-pytest==7.4.4
-    # via -r image_vec_rep_resnet_requirements.in
-regex==2023.12.25
-    # via nltk
-textblob==0.17.1
+    #   torchvision
+pillow==10.2.0
+    # via
+    #   -r image_vec_rep_resnet_requirements.in
+    #   torchvision
+requests==2.31.0
+    # via torchvision
+sympy==1.12
+    # via torch
+torch==2.1.2+cpu
+    # via
+    #   -r image_vec_rep_resnet_requirements.in
+    #   torchvision
+torchvision==0.16.2+cpu
     # via -r image_vec_rep_resnet_requirements.in
-tqdm==4.66.1
-    # via nltk
+typing-extensions==4.9.0
+    # via torch
+urllib3==2.2.0
+    # via requests

src/api/core/operators/text_vec_rep_paraphrase_lxml_requirements.in

@@ -1,5 +1,5 @@
 numpy==1.26.3
-nltk==3.6.6
+nltk==3.8.1
 scikit-learn==1.3.2
 scipy==1.11.4
 sentence-transformers==2.2.2

src/api/core/operators/text_vec_rep_paraphrase_lxml_requirements.txt

@@ -40,7 +40,7 @@ mpmath==1.3.0
     # via sympy
 networkx==3.2.1
     # via torch
-nltk==3.6.6
+nltk==3.8.1
     # via
     #   -r text_vec_rep_paraphrase_lxml_requirements.in
     #   sentence-transformers

src/api/core/operators/vid_vec_rep_resnet.py

@@ -3,7 +3,7 @@
 def initialize(param):
     print("Installing packages for vid_vec_rep_resnet")
 
-    global TextBlob, os, np, cv2, qr, torch, data, models, transforms, Image, wget #, FFmpeg
+    global os, np, cv2, qr, torch, data, models, transforms, Image #, wget #, FFmpeg
     global imagenet_transform, ImageListDataset, VideoAnalyzer, gendata #, compress_video
 
     import os
@@ -16,7 +16,7 @@ def initialize(param):
     import torchvision.transforms as transforms
     from PIL import Image
     # from ffmpy import FFmpeg
-    import wget
+    # import wget
 
     imagenet_transform = transforms.Compose(
         [
@@ -88,7 +88,7 @@ def __init__(self, video, sampling_rate=10, n_keyframes=5):
             self.n_samples = self.n_frames / sampling_rate
             self.n_keyframes = n_keyframes
             # print("init model")
-            self.model = models.resnet18(pretrained=True)
+            self.model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
             # print(type(self.model))
             # list of individual PIL Images
             self.frame_images = []

src/api/core/operators/vid_vec_rep_resnet_requirements.in

@@ -1,7 +1,6 @@
 torch==2.1.2+cpu
 torchvision==0.16.2+cpu
 numpy==1.26.3
-wget==3.2
 Pillow==10.2.0
 scipy==1.11.4
 opencv-python-headless==4.9.0.80

src/api/core/operators/vid_vec_rep_resnet_requirements.txt

@@ -52,5 +52,3 @@ typing-extensions==4.9.0
     # via torch
 urllib3==2.0.7
     # via requests
-wget==3.2
-    # via -r vid_vec_rep_resnet_requirements.in

src/api/requirements.in

@@ -2,24 +2,16 @@ flask==2.3.2
 flask_cors==3.0.9
 google-cloud==0.34.0
 google-cloud-vision==3.5.0
-scikit-image==0.22.0
-tqdm==4.66.1
-torch==2.1.2+cpu
-torchvision==0.16.2+cpu
-textblob==0.17.1
 Pillow==10.2.0
-opencv-python-headless==4.9.0.80
 elasticsearch==8.11.1
 wget==3.2
 pika==1.3.2
-ffmpy==0.3.1
-sentence-transformers==2.2.2
 python-dotenv==1.0.0
-boto3==1.34.19
+boto3==1.34.34
 click==8.1.7
-typing-extensions==4.9.0
 pytest==7.4.4
 PyYAML==6.0.1
 dacite==1.8.1
 memray==1.11.0  # dev
 pyinstrument==4.6.2
+numpy==1.26.3