From 810a45eae4b681799173286b683e64b2d387e38f Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Sat, 23 Mar 2024 13:36:30 +0530 Subject: [PATCH] fix: Added pinned pip package - Added base requirements.in with pip - Created base requirements.txt with pip pinned by hash - Recreated requirements.txt with --allow-unsafe to get correct setuptools - Updated feluda core dockerfile to install pip using base requirements - Updated ci test dockerfile to install pip using base requirements - Updated all benchmark dockerfiles to install pip using base requirements - Updated all worker dockerfiles to install pip using base requirements - Updated readme pip install and pip-compile commands --- README.md | 14 +++++++------- src/Dockerfile | 5 ++++- src/Dockerfile.test | 5 +++-- src/base_requirements.in | 1 + src/base_requirements.txt | 12 ++++++++++++ .../audiovec/Dockerfile.audio_vec_embedding | 4 +++- .../Dockerfile.audio_vec_embedding_graviton | 4 +++- .../imgvec/Dockerfile.image_vec_rep_resnet | 5 ++++- src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet | 5 ++++- .../vidvec/Dockerfile.vid_vec_rep_resnet_graviton | 4 +++- src/requirements.txt | 13 ++++++++----- src/worker/audiovec/Dockerfile.audio_worker | 5 +++-- .../audiovec/Dockerfile.audio_worker_graviton | 5 +++-- src/worker/hash/Dockerfile.hash_worker | 5 +++-- src/worker/hash/Dockerfile.hash_worker_graviton | 5 +++-- src/worker/vidvec/Dockerfile.video_worker | 5 +++-- src/worker/vidvec/Dockerfile.video_worker_graviton | 5 +++-- 17 files changed, 70 insertions(+), 32 deletions(-) create mode 100644 src/base_requirements.in create mode 100644 src/base_requirements.txt diff --git a/README.md b/README.md index 03cbde43..3d0434d7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Please create a new Discussion [here](https://github.com/tattle-made/tattle-api/ ``` # Install locally in venv $ cd src/api/ - $ pip install -r requirements.txt + $ pip install --require-hashes --no-deps -r requirements.txt ``` @@ -67,8 +67,8 @@ Please create a new Discussion [here](https://github.com/tattle-made/tattle-api/ ``` # Install locally in venv $ cd src/api/core/operators/ - $ pip install -r image_vec_rep_resnet_requirements.txt - $ pip install -r vid_vec_rep_resnet_requirements.txt + $ pip install --require-hashes --no-deps -r image_vec_rep_resnet_requirements.txt + $ pip install --require-hashes --no-deps -r vid_vec_rep_resnet_requirements.txt .. # Create the docker containers $ cd src/api/ @@ -127,13 +127,13 @@ Note: ```bash $ cd src/ $ pip install --upgrade pip-tools -$ TMPDIR= pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links requirements.in +$ TMPDIR= pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links requirements.in # Updating operators $ cd src/core/operators/ # The link for torch is required since PyPi only hosts the GPU version of torch packages. -$ TMPDIR= pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html vid_vec_rep_resnet_requirements.in -$ TMPDIR= pip-compile --verbose --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html audio_vec_embedding_requirements.in +$ TMPDIR= pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html vid_vec_rep_resnet_requirements.in +$ TMPDIR= pip-compile --verbose --allow-unsafe --generate-hashes --emit-index-url --emit-find-links --find-links https://download.pytorch.org/whl/torch_stable.html audio_vec_embedding_requirements.in ``` #### Modify generated `requirements.txt` for platform specific torch packages @@ -199,7 +199,7 @@ torchvision==0.17.0+cpu; platform_machine=='x86_64' \ This is useful to update dependencies e.g. when using `pip-audit` ```bash -$ TMPDIR= pip-compile --verbose --generate-hashes --find-links https://download.pytorch.org/whl/torch_stable.html --upgrade-package == --upgrade-package +$ TMPDIR= pip-compile --verbose --allow-unsafe --generate-hashes --find-links https://download.pytorch.org/whl/torch_stable.html --upgrade-package == --upgrade-package ``` diff --git a/src/Dockerfile b/src/Dockerfile index 5d57e22d..191e60c0 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -26,7 +26,10 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt + RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends vim curl # RUN apt-get install -y ffmpeg # RUN apt-get update && \ diff --git a/src/Dockerfile.test b/src/Dockerfile.test index 7142b781..73cdfff9 100644 --- a/src/Dockerfile.test +++ b/src/Dockerfile.test @@ -24,13 +24,14 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt diff --git a/src/base_requirements.in b/src/base_requirements.in new file mode 100644 index 00000000..14d3f4de --- /dev/null +++ b/src/base_requirements.in @@ -0,0 +1 @@ +pip==24.0 \ No newline at end of file diff --git a/src/base_requirements.txt b/src/base_requirements.txt new file mode 100644 index 00000000..2374f317 --- /dev/null +++ b/src/base_requirements.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes base_requirements.in +# + +# The following packages are considered to be unsafe in a requirements file: +pip==24.0 \ + --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \ + --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2 + # via -r base_requirements.in diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding index cd3d9118..e68a40b8 100644 --- a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding +++ b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding @@ -20,7 +20,9 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt # audio requirments file COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton index 9a91c65f..86f5ac2c 100644 --- a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton +++ b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton @@ -20,7 +20,9 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt # audio requirments file COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt diff --git a/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet b/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet index c84f758d..1eb9fe66 100644 --- a/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet +++ b/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet @@ -23,7 +23,10 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt + COPY --chown=python:python ./core/operators/image_vec_rep_resnet_requirements.txt /usr/app/core/operators/image_vec_rep_resnet_requirements.txt RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/core/operators/image_vec_rep_resnet_requirements.txt COPY --chown=python:python ./core/operators/image_vec_rep_resnet.py /usr/app/core/operators/image_vec_rep_resnet.py diff --git a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet index f68340d2..68d4e41b 100644 --- a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet +++ b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet @@ -20,7 +20,10 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt + # video requirments file COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt diff --git a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton index e1c110c8..deb451d1 100644 --- a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton +++ b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton @@ -20,7 +20,9 @@ WORKDIR /usr/app RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv ENV PATH="/usr/app/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip +# base requirements file +COPY --chown=python:python base_requirements.txt /usr/app/base_requirements.txt +RUN pip install --no-cache-dir --require-hashes --no-deps -r /usr/app/base_requirements.txt ### AWS Graviton Optimization ### diff --git a/src/requirements.txt b/src/requirements.txt index 7243e974..c0be9610 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --generate-hashes requirements.in +# pip-compile --allow-unsafe --generate-hashes requirements.in # blinker==1.7.0 \ --hash=sha256:c3f865d4d54db7abc53758a01601cf343fe55b84c1de4e3fa910e420b438d5b9 \ @@ -1203,7 +1203,10 @@ zope-interface==6.1 \ --hash=sha256:fddbab55a2473f1d3b8833ec6b7ac31e8211b0aa608df5ab09ce07f3727326de # via gevent -# WARNING: The following packages were not pinned, but pip requires them to be -# pinned when the requirements file includes hashes and the requirement is not -# satisfied by a package already installed. Consider using the --allow-unsafe flag. -# setuptools +# The following packages are considered to be unsafe in a requirements file: +setuptools==69.2.0 \ + --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ + --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c + # via + # zope-event + # zope-interface diff --git a/src/worker/audiovec/Dockerfile.audio_worker b/src/worker/audiovec/Dockerfile.audio_worker index 40982e02..192a716d 100644 --- a/src/worker/audiovec/Dockerfile.audio_worker +++ b/src/worker/audiovec/Dockerfile.audio_worker @@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt diff --git a/src/worker/audiovec/Dockerfile.audio_worker_graviton b/src/worker/audiovec/Dockerfile.audio_worker_graviton index 82480406..85452457 100644 --- a/src/worker/audiovec/Dockerfile.audio_worker_graviton +++ b/src/worker/audiovec/Dockerfile.audio_worker_graviton @@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /home/python/app/core/operators/audio_vec_embedding_requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/audio_vec_embedding_requirements.txt diff --git a/src/worker/hash/Dockerfile.hash_worker b/src/worker/hash/Dockerfile.hash_worker index 4596a8d4..9a0776ba 100644 --- a/src/worker/hash/Dockerfile.hash_worker +++ b/src/worker/hash/Dockerfile.hash_worker @@ -25,11 +25,12 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt ##################################### diff --git a/src/worker/hash/Dockerfile.hash_worker_graviton b/src/worker/hash/Dockerfile.hash_worker_graviton index 786c0b98..cf416ae2 100644 --- a/src/worker/hash/Dockerfile.hash_worker_graviton +++ b/src/worker/hash/Dockerfile.hash_worker_graviton @@ -25,11 +25,12 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt ##################################### diff --git a/src/worker/vidvec/Dockerfile.video_worker b/src/worker/vidvec/Dockerfile.video_worker index b81c7283..89edf5c3 100644 --- a/src/worker/vidvec/Dockerfile.video_worker +++ b/src/worker/vidvec/Dockerfile.video_worker @@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt diff --git a/src/worker/vidvec/Dockerfile.video_worker_graviton b/src/worker/vidvec/Dockerfile.video_worker_graviton index a8cf3ca1..8d6ab69f 100644 --- a/src/worker/vidvec/Dockerfile.video_worker_graviton +++ b/src/worker/vidvec/Dockerfile.video_worker_graviton @@ -25,12 +25,13 @@ RUN python -m venv /home/python/app/venv \ # Set venv in path ENV PATH="/home/python/app/venv/bin:$PATH" -# Copy core and operator requirements +# Copy base, core and operator requirements +COPY --chown=python:python base_requirements.txt /home/python/app/base_requirements.txt COPY --chown=python:python requirements.txt /home/python/app/requirements.txt COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt # Run pip install -RUN pip install --no-cache-dir --upgrade pip \ +RUN pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/base_requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/requirements.txt \ && pip install --no-cache-dir --require-hashes --no-deps -r /home/python/app/core/operators/vid_vec_rep_resnet_requirements.txt