MP-1215 Add images without conda/pip dependencies (#19)

This PR contains changes for the base images, which were needed for migrating vid-vision to uv: * Since we have upgraded pytorch, we don't have to install an older gcc anymore, so I could switch base Debian images from Bullseye to Bookworm (11 -> 12). * opensc had to be upgraded from 0.20.0 to 0.23.0 to work in newer Debian. * I don't replace old conda-based images, but add new ones (we may still need old images somewhere). * We always use a prebuilt pytorch now, even for cpu images. * genicam images have been removed. * For uv there are no separate pytorch images now: they are merged with moonbox images, since we don't do any python installations now. * All python dependencies have been moved to corresponding repositories; base images do not contain them anymore. Notes: * I didn't try to optimize installing of apt packages except some cases, when I had to do changes for adding uv anyway. * I could not completely get rid of conda: there were some packages, which otherwise have some complicated installation procedure - and I don't know, what exactly is needed from them (i.e., which parts should be installed, with which parameters should they be built etc.).
MoonVision · Dec 4, 2024 · 99c1ded · 99c1ded
1 parent 22a15df
commit 99c1ded
Show file tree

Hide file tree

Showing 13 changed files with 245 additions and 75 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,15 +24,37 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}python-base
         path: docker/python-base
         dockerfile: docker/python-base/Dockerfile
-        baseimage: mambaorg/micromamba:1-bullseye
+        baseimage: mambaorg/micromamba:1
     - uses: ./.github/create-docker
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}python-base
         path: docker/python-base
         dockerfile: docker/python-base/Dockerfile
-        baseimage: mambaorg/micromamba:1-bullseye
+        baseimage: mambaorg/micromamba:1
         additionaltag: 3.11
         additional_buildargs: env_file=environment_3.11.yml
+  docker-python-base-uv:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/create-docker
+        with:
+          imagename: moonvision/${{ env.IMAGE_PRE }}python-base
+          path: docker/python-base
+          dockerfile: docker/python-base/DockerfileUV
+          baseimage: mambaorg/micromamba:1
+          additionaltag: uv
+          additional_buildargs: python_version=3.8
+      - uses: ./.github/create-docker
+        with:
+          imagename: moonvision/${{ env.IMAGE_PRE }}python-base
+          path: docker/python-base
+          dockerfile: docker/python-base/DockerfileUV
+          baseimage: mambaorg/micromamba:1
+          additionaltag: uv-3.11
+          additional_buildargs: |-
+            env_file=environment_uv.yml
+            python_version=3.11
   build-ffmpeg:
     runs-on: ubuntu-latest
     steps:
@@ -44,7 +66,7 @@ jobs:
         tagversion: 4.2.1
         path: docker/builders/ffmpeg
         dockerfile: docker/builders/ffmpeg/Dockerfile
-        baseimage: mambaorg/micromamba:1-bullseye
+        baseimage: mambaorg/micromamba:1
         additional_buildargs: ffmpeg_version=4.2.1
   build-ffmpeg-cuda:
     runs-on: ubuntu-latest
@@ -55,7 +77,7 @@ jobs:
         imagename: moonvision/${{ env.IMAGE_PRE }}custom-builds
         additionaltag: ffmpeg-cuda
         tagversion: 4.2.1
-        baseimage: mambaorg/micromamba:1-bullseye
+        baseimage: mambaorg/micromamba:1
         path: docker/builders/ffmpeg
         dockerfile: docker/builders/ffmpeg/Dockerfile
         additional_buildargs: |-
@@ -72,7 +94,7 @@ jobs:
         tagversion: 4.14
         path: docker/builders/proxychains
         dockerfile: docker/builders/proxychains/Dockerfile
-        baseimage: debian:bullseye-slim
+        baseimage: debian:bookworm-slim
         additional_buildargs: proxychains_version=4.14
   build-opensc:
     runs-on: ubuntu-latest
@@ -82,11 +104,11 @@ jobs:
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}custom-builds
         additionaltag: opensc
-        tagversion: 0.20.0
+        tagversion: 0.23.0
         path: docker/builders/opensc
         dockerfile: docker/builders/opensc/Dockerfile
-        baseimage: debian:bullseye-slim
-        additional_buildargs: opensc_version=0.20.0
+        baseimage: debian:bookworm-slim
+        additional_buildargs: opensc_version=0.23.0
   build-pytorch:
     needs:
     - docker-python-base
@@ -104,7 +126,7 @@ jobs:
         additional_buildargs: |-
           pytorch_tag=v2.0.1
           torchvision_tag=v0.15.2
-          prebuilt=false
+          prebuilt=true
   build-pytorch-311:
     needs:
     - docker-python-base
@@ -122,7 +144,7 @@ jobs:
         additional_buildargs: |-
           pytorch_tag=v2.0.1
           torchvision_tag=v0.15.2
-          prebuilt=false
+          prebuilt=true
   build-pytorch-cuda:
     needs:
     - docker-python-base
@@ -197,81 +219,107 @@ jobs:
           additional_buildargs: |-
             ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
             pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-3.11-2.0.1_torchvision-0.15.2
-  docker-moonbox-genicam:
+  docker-moonbox-cuda:
     needs:
     - docker-python-base
-    - build-ffmpeg
-    - build-pytorch
+    - build-ffmpeg-cuda
+    - build-pytorch-cuda
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/create-docker
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
-        additionaltag: genicam
+        additionaltag: cuda
         path: docker/moonbox
         dockerfile: docker/moonbox/Dockerfile
         baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest
         additional_buildargs: |-
-          with_genicam=true
+          with_cuda=true
+          ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1
+          pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-cuda-2.0.1_torchvision-0.15.2
+  docker-moonbox-cuda-311:
+    needs:
+    - docker-python-base
+    - build-ffmpeg-cuda
+    - build-pytorch-cuda-311
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/create-docker
+      with:
+        imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
+        additionaltag: cuda-3.11
+        path: docker/moonbox
+        dockerfile: docker/moonbox/Dockerfile
+        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest
+        additional_buildargs: |-
+          with_cuda=true
+          ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1
+          pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-cuda-3.11-2.0.1_torchvision-0.15.2
+  docker-moonbox-mini-uv:
+    needs:
+    - docker-python-base-uv
+    - build-ffmpeg
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ./.github/create-docker
+      with:
+        imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
+        additionaltag: mini-uv
+        path: docker/moonbox
+        dockerfile: docker/moonbox/DockerfileUV
+        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest
+        additional_buildargs: |-
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
-          pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-2.0.1_torchvision-0.15.2
-  docker-moonbox-genicam-311:
+  docker-moonbox-mini-uv-311:
     needs:
-      - docker-python-base
+      - docker-python-base-uv
       - build-ffmpeg
-      - build-pytorch-311
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/create-docker
         with:
           imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
-          additionaltag: genicam-3.11
+          additionaltag: mini-uv-3.11
           path: docker/moonbox
-          dockerfile: docker/moonbox/Dockerfile
-          baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest
+          dockerfile: docker/moonbox/DockerfileUV
+          baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest
           additional_buildargs: |-
-            with_genicam=true
             ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-4.2.1
-            pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-3.11-2.0.1_torchvision-0.15.2
-  docker-moonbox-cuda:
+  docker-moonbox-cuda-uv:
     needs:
-    - docker-python-base
+    - docker-python-base-uv
     - build-ffmpeg-cuda
-    - build-pytorch-cuda
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/create-docker
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
-        additionaltag: cuda
+        additionaltag: cuda-uv
         path: docker/moonbox
-        dockerfile: docker/moonbox/Dockerfile
-        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:latest
+        dockerfile: docker/moonbox/DockerfileUV
+        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-latest
         additional_buildargs: |-
-          with_genicam=false
           with_cuda=true
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1
-          pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-cuda-2.0.1_torchvision-0.15.2
-  docker-moonbox-cuda-311:
+  docker-moonbox-cuda-uv-311:
     needs:
-    - docker-python-base
+    - docker-python-base-uv
     - build-ffmpeg-cuda
-    - build-pytorch-cuda-311
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/create-docker
       with:
         imagename: moonvision/${{ env.IMAGE_PRE }}moonbox
-        additionaltag: cuda-3.11
+        additionaltag: cuda-uv-3.11
         path: docker/moonbox
-        dockerfile: docker/moonbox/Dockerfile
-        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:3.11-latest
+        dockerfile: docker/moonbox/DockerfileUV
+        baseimage: moonvision/${{ env.IMAGE_PRE }}python-base:uv-3.11-latest
         additional_buildargs: |-
-          with_genicam=false
           with_cuda=true
           ffmpeg_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:ffmpeg-cuda-4.2.1
-          pytorch_from_docker=moonvision/${{ env.IMAGE_PRE }}custom-builds:pytorch-cuda-3.11-2.0.1_torchvision-0.15.2
diff --git a/Readme.md b/Readme.md
@@ -20,7 +20,7 @@ Python Base from a [micromamba](https://github.com/mamba-org/mamba#micromamba) e
 
 | Build arg | Default | Description |
 |-|-|-|
-| `baseimage` | `mambaorg/micromamba:1.0.0-bullseye` | Set baseimage from which python base is built. |
+| `baseimage` | `mambaorg/micromamba:1.0.0` | Set baseimage from which python base is built. |
 
 #### Python Base
 
@@ -39,7 +39,6 @@ docker build \
 | `baseimage` | `moonvision/python-base` | Set the baseimage from which moonbox is built. |
 | `ffmpeg_from_docker` | `moonvision/custom-builds:ffmpeg-4.2.1` | The docker image from which FFmpeg will be installed. |
 | `pytorch_from_docker` | `moonvision/custom-builds:pytorch-1.11.0_torchvision-0.12.0` | The docker image from which PyTorch and Torchvision will be installed. |
-| `with_genicam` | `false` | Whether to install GeniCam harvesters. |
 | `with_cuda` | `false` / undefined | If `true` install CUDA specific packages. If defined, set ENV variables to mount nvidia binaries at runtime. |
 
 #### Moonbox Mini
@@ -52,15 +51,6 @@ docker build \
   -f docker/moonbox/Dockerfile docker/moonbox
 ```
 
-#### Moonbox with GeniCam
-
-```
-docker build \
-  -t moonvision/moonbox:genicam-latest \
-  --build-arg with_genicam=true \
-  -f docker/moonbox/Dockerfile docker/moonbox
-```
-
 #### Moonbox CUDA
 
 ```
@@ -81,7 +71,7 @@ A custom build of [FFmpeg](https://ffmpeg.org).
 
 | Build arg | Default | Description |
 |-|-|-|
-| `baseimage` | `mambaorg/micromamba:1-bullseye` | Set the image in which FFmpeg will be built. |
+| `baseimage` | `mambaorg/micromamba:1` | Set the image in which FFmpeg will be built. |
 | `ffmpeg_version` | | The FFmpeg release version to build. |
 | `with_cuda` | `false` | Whether to build FFmpeg with Nvidia hardware acceleration.  |
 

diff --git a/docker/builders/ffmpeg/Dockerfile b/docker/builders/ffmpeg/Dockerfile
@@ -1,4 +1,4 @@
-ARG baseimage='mambaorg/micromamba:1-bullseye'
+ARG baseimage='mambaorg/micromamba:1'
 
 FROM ${baseimage} AS builder
 

diff --git a/docker/builders/opensc/Dockerfile b/docker/builders/opensc/Dockerfile
@@ -1,4 +1,4 @@
-ARG baseimage='debian:bullseye-slim'
+ARG baseimage='debian:bookworm-slim'
 
 FROM $baseimage AS builder
 
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y autoconf automake build-essential \
     checkinstall docbook-xsl libccid libpcsclite-dev libreadline-dev \
     libssl-dev libtool pcscd pkg-config wget xsltproc
 
-ARG opensc_version=0.20.0
+ARG opensc_version=0.23.0
 
 # Fetch Tarball
 RUN mkdir -p ~/temp \

diff --git a/docker/builders/proxychains/Dockerfile b/docker/builders/proxychains/Dockerfile
@@ -1,4 +1,4 @@
-ARG baseimage='debian:bullseye-slim'
+ARG baseimage='debian:bookworm-slim'
 
 FROM $baseimage AS builder
 

diff --git a/docker/moonbox/Dockerfile b/docker/moonbox/Dockerfile
@@ -7,15 +7,14 @@ FROM $pytorch_from_docker as pytorch_builder
 
 FROM ${baseimage} as installer
 
-ARG with_genicam='false'
 ARG with_cuda='false'
 
 COPY --from=ffmpeg_builder /packages /ffmpeg-packages
 COPY --from=pytorch_builder /packages /pytorch-packages
 COPY . /bd_build
 
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
-RUN with_genicam=$with_genicam with_cuda=$with_cuda bash /bd_build/install_deps.sh
+RUN with_cuda=$with_cuda bash /bd_build/install_deps.sh
 
 # squash /packages
 FROM ${baseimage}

diff --git a/docker/moonbox/DockerfileUV b/docker/moonbox/DockerfileUV
@@ -0,0 +1,31 @@
+ARG baseimage='moonvision/python-base:uv-latest'
+ARG ffmpeg_from_docker='moonvision/custom-builds:ffmpeg-4.2.1'
+
+FROM $ffmpeg_from_docker as ffmpeg_builder
+
+FROM ${baseimage} as installer
+
+ARG with_cuda='false'
+
+ARG DEBIAN_FRONTEND='noninteractive'
+ # fixme: Why should gcc be installed both here and in python-base? And the same for checkinstall.
+RUN apt-get update --yes --no-install-recommends \
+ && apt-get install --yes gcc g++ checkinstall cmake --no-install-recommends
+
+COPY --from=ffmpeg_builder /packages /ffmpeg-packages
+COPY . /bd_build
+
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+RUN with_cuda=$with_cuda bash /bd_build/install_deps_uv.sh
+
+# squash /packages
+FROM ${baseimage}
+COPY --from=installer / /
+
+RUN echo "ffmpeg codecs: " && ffmpeg -codecs
+
+# convince nvidia runc shim to inject driver, c.f.:
+# https://canvatechblog.com/supporting-gpu-accelerated-machine-learning-with-kubernetes-and-nix-7c1da8e42f61
+ARG with_cuda
+ENV NVIDIA_VISIBLE_DEVICES=${with_cuda:+all}
+ENV NVIDIA_DRIVER_CAPABILITIES=${with_cuda:+compute,utility}
diff --git a/docker/moonbox/install_deps.sh b/docker/moonbox/install_deps.sh
@@ -29,14 +29,6 @@ rm -rf /ffmpeg-packages
 pip install --no-cache-dir /pytorch-packages/*.whl -c /bd_build/constraints.txt
 rm -rf /pytorch-packages
 
-echo "with_genicam: ${with_genicam}, with_cuda: ${with_cuda}"
-
-if test "$with_genicam" = "true"; then
-    bash /bd_build/install_genicam.sh
-else
-    echo 'Skip GeniCam'
-fi
-
 if test "$with_cuda" = "true"; then
     bash /bd_build/install_cuda.sh
 else