Sample Docker Push Action

pytorch · Jul 24, 2023 · 365de49 · 365de49
1 parent 6920876
commit 365de49
Show file tree

Hide file tree

Showing 3 changed files with 149 additions and 60 deletions.
diff --git a/.github/workflows/docker_builder.yml b/.github/workflows/docker_builder.yml
@@ -0,0 +1,80 @@
+# name: 'Torch-TRT Docker'
+
+# on:
+#   pull_request:
+#     types: [opened, synchronize, ready_for_review, review_requested, reopened]
+
+# ##### Adapted from https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+
+# # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+# env:
+#   REGISTRY: ghcr.io
+#   IMAGE_NAME: ${{ github.repository }}
+
+# # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+# jobs:
+#   build-and-push-image:
+#     runs-on: ubuntu-latest
+#     # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+#     permissions:
+#       contents: read
+#       packages: write
+#     steps:
+#       - name: Checkout repository
+#         uses: actions/checkout@v3
+#       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+#       - name: Log in to the Container registry
+#         uses: docker/login-action@v2
+#         with:
+#           registry: ${{ env.REGISTRY }}
+#           username: ${{ github.actor }}
+#           password: ${{ secrets.GITHUB_TOKEN }}
+
+#       - name: Reclaim space
+#         run: |
+#           rm -rf /usr/share/dotnet
+#           rm -rf /opt/ghc
+#           rm -rf "/usr/local/share/boost"
+#           rm -rf /usr/local/cuda/cuda-*
+
+#       - name: ssh
+#         uses: appleboy/ssh-action@master
+#         with:
+#           host: ${{ secrets.HOST }}
+#           username: ${{ secrets.USERNAME }}
+#           password: ${{ secrets.PASSWORD }}
+#           port: ${{ secrets.PORT }}
+
+#       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+#       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+#       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+#       - name: Build and push Docker image
+#         uses: docker/build-push-action@v4
+#         with:
+#           context: .
+#           push: true
+#           tags: torch_tensorrt:main
+#           file: docker/Dockerfile
+#           build-args: |
+#             TENSORRT_VERSION=8.6
+#             CUDNN_VERSION=8.8
+#           shm-size: 4g
+
+
+name: 'Torch-TRT Docker'
+
+on:
+  pull_request:
+    types: [opened, synchronize, ready_for_review, review_requested, reopened]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Build the Docker image
+      run: DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=8.6 --build-arg CUDNN_VERSION=8.8 -f docker/Dockerfile --tag torch_tensorrt_main:$(date +%s) .
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -18,8 +18,8 @@ ENV USE_CXX11=${USE_CXX11_ABI}
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install basic dependencies
-RUN apt-get update
-RUN apt install -y build-essential manpages-dev wget zlib1g software-properties-common git libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget ca-certificates curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8
+# RUN apt-get update
+# RUN apt install -y build-essential manpages-dev wget zlib1g software-properties-common git libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget ca-certificates curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8
 
 # Install PyEnv and desired Python version
 ENV HOME="/root"
@@ -33,82 +33,90 @@ RUN wget -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-instal
 RUN pyenv install -v ${PYTHON_VERSION}
 RUN pyenv global ${PYTHON_VERSION}
 
+RUN pip install tensorrt
+
 # Install CUDNN + TensorRT + dependencies
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin
-RUN mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/7fa2af80.pub
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 536F8F1DE80F6A35
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
-RUN add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /"
-RUN apt-get update
-RUN apt-get install -y libcudnn8=${CUDNN_VERSION}* libcudnn8-dev=${CUDNN_VERSION}*
+# RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin
+# RUN mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600
+# RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/7fa2af80.pub
+# RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 536F8F1DE80F6A35
+# RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC
+# RUN add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /"
+# RUN apt-get update
+# RUN apt-get install -y libcudnn8=${CUDNN_VERSION}* libcudnn8-dev=${CUDNN_VERSION}*
+
+# RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
+# RUN add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /"
+# RUN apt-get update
 
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
-RUN add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /"
-RUN apt-get update
+# RUN apt-get install -y libnvinfer8=${TENSORRT_VERSION}.* libnvinfer-plugin8=${TENSORRT_VERSION}.* libnvinfer-dev=${TENSORRT_VERSION}.* libnvinfer-plugin-dev=${TENSORRT_VERSION}.* libnvonnxparsers8=${TENSORRT_VERSION}.* libnvonnxparsers-dev=${TENSORRT_VERSION}.* libnvparsers8=${TENSORRT_VERSION}.*  libnvparsers-dev=${TENSORRT_VERSION}.*
 
-RUN apt-get install -y libnvinfer8=${TENSORRT_VERSION}.* libnvinfer-plugin8=${TENSORRT_VERSION}.* libnvinfer-dev=${TENSORRT_VERSION}.* libnvinfer-plugin-dev=${TENSORRT_VERSION}.* libnvonnxparsers8=${TENSORRT_VERSION}.* libnvonnxparsers-dev=${TENSORRT_VERSION}.* libnvparsers8=${TENSORRT_VERSION}.*  libnvparsers-dev=${TENSORRT_VERSION}.*
+# # Setup Bazel via Bazelisk
+# RUN wget -q https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 -O /usr/bin/bazel &&\
+#     chmod a+x /usr/bin/bazel
 
-# Setup Bazel via Bazelisk
-RUN wget -q https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 -O /usr/bin/bazel &&\
-    chmod a+x /usr/bin/bazel
+# # Build Torch-TensorRT in an auxillary container
+# FROM base as torch-tensorrt-builder-base
 
-# Build Torch-TensorRT in an auxillary container
-FROM base as torch-tensorrt-builder-base
+# ARG ARCH="x86_64"
+# ARG TARGETARCH="amd64"
 
-ARG ARCH="x86_64"
-ARG TARGETARCH="amd64"
+# RUN apt-get install -y python3-setuptools
+# RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
 
-RUN apt-get install -y python3-setuptools
-RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub
+# RUN apt-get update &&\
+#     apt-get install -y --no-install-recommends locales ninja-build &&\
+#     rm -rf /var/lib/apt/lists/* &&\
+#     locale-gen en_US.UTF-8
 
-RUN apt-get update &&\
-    apt-get install -y --no-install-recommends locales ninja-build &&\
-    rm -rf /var/lib/apt/lists/* &&\
-    locale-gen en_US.UTF-8
+# FROM torch-tensorrt-builder-base as torch-tensorrt-builder
 
-FROM torch-tensorrt-builder-base as torch-tensorrt-builder
+# COPY . /workspace/torch_tensorrt/src
+# WORKDIR /workspace/torch_tensorrt/src
+# RUN cp ./docker/WORKSPACE.docker WORKSPACE
 
-COPY . /workspace/torch_tensorrt/src
-WORKDIR /workspace/torch_tensorrt/src
-RUN cp ./docker/WORKSPACE.docker WORKSPACE
+# # Symlink the path pyenv is using for python with the /opt directory for package sourcing
+# RUN mkdir -p "/opt/python3/" &&\
+#     ln -s "`pyenv which python | xargs dirname | xargs dirname`/lib/python$PYTHON_VERSION/site-packages" "/opt/python3/"
 
-# Symlink the path pyenv is using for python with the /opt directory for package sourcing
-RUN mkdir -p "/opt/python3/" &&\
-    ln -s "`pyenv which python | xargs dirname | xargs dirname`/lib/python$PYTHON_VERSION/site-packages" "/opt/python3/"
+# RUN rm -rf /usr/share/dotnet
+# RUN rm -rf /opt/ghc
+# RUN rm -rf "/usr/local/share/boost"
+# RUN rm -rf /usr/local/cuda/cuda-*
+# RUN apt-get clean
 
-# Extract base image cuda version (everything after :, before -, before final ., in BASE_IMG)
-# Ensure the default cuda folder agrees with the version in the base image
-RUN CUDA_BASE_IMG_VERSION_INTERMEDIATE=`echo ${BASE_IMG#*:}` &&\
-    CUDA_BASE_IMG_VERSION=`echo ${CUDA_BASE_IMG_VERSION_INTERMEDIATE%%-*}` &&\
-    CUDA_MAJOR_MINOR_VERSION=`echo ${CUDA_BASE_IMG_VERSION%.*}` &&\
-    rm -fr /usr/local/cuda &&\
-    ln -s /usr/local/cuda-${CUDA_MAJOR_MINOR_VERSION} /usr/local/cuda
+# # Extract base image cuda version (everything after :, before -, before final ., in BASE_IMG)
+# # Ensure the default cuda folder agrees with the version in the base image
+# RUN CUDA_BASE_IMG_VERSION_INTERMEDIATE=`echo ${BASE_IMG#*:}` &&\
+#     CUDA_BASE_IMG_VERSION=`echo ${CUDA_BASE_IMG_VERSION_INTERMEDIATE%%-*}` &&\
+#     CUDA_MAJOR_MINOR_VERSION=`echo ${CUDA_BASE_IMG_VERSION%.*}` &&\
+#     rm -fr /usr/local/cuda &&\
+#     ln -s /usr/local/cuda-${CUDA_MAJOR_MINOR_VERSION} /usr/local/cuda
 
-ENV CUDA_HOME=/usr/local/cuda
+# ENV CUDA_HOME=/usr/local/cuda
 
-# This script builds both libtorchtrt bin/lib/include tarball and the Python wheel, in dist/
-RUN bash ./docker/dist-build.sh
+# # This script builds both libtorchtrt bin/lib/include tarball and the Python wheel, in dist/
+# RUN bash ./docker/dist-build.sh
 
-# Copy and install Torch-TRT into the main container
-FROM base as torch-tensorrt
+# # Copy and install Torch-TRT into the main container
+# FROM base as torch-tensorrt
 
-COPY . /opt/torch_tensorrt
+# COPY . /opt/torch_tensorrt
 
-# Symlink the path pyenv is using for python with the /opt directory for package sourcing
-RUN mkdir -p "/opt/python3/" &&\
-    ln -s "`pyenv which python | xargs dirname | xargs dirname`/lib/python$PYTHON_VERSION/site-packages" "/opt/python3/"
+# # Symlink the path pyenv is using for python with the /opt directory for package sourcing
+# RUN mkdir -p "/opt/python3/" &&\
+#     ln -s "`pyenv which python | xargs dirname | xargs dirname`/lib/python$PYTHON_VERSION/site-packages" "/opt/python3/"
 
-COPY --from=torch-tensorrt-builder  /workspace/torch_tensorrt/src/py/dist/ .
+# COPY --from=torch-tensorrt-builder  /workspace/torch_tensorrt/src/py/dist/ .
 
-RUN cp /opt/torch_tensorrt/docker/WORKSPACE.docker /opt/torch_tensorrt/WORKSPACE
-RUN pip install -r /opt/torch_tensorrt/py/requirements.txt
-RUN pip install tensorrt==${TENSORRT_VERSION}.*
-RUN pip install *.whl && rm -fr /workspace/torch_tensorrt/py/dist/* *.whl
+# RUN cp /opt/torch_tensorrt/docker/WORKSPACE.docker /opt/torch_tensorrt/WORKSPACE
+# RUN pip install -r /opt/torch_tensorrt/py/requirements.txt
+# RUN pip install tensorrt==${TENSORRT_VERSION}.*
+# RUN pip install *.whl && rm -fr /workspace/torch_tensorrt/py/dist/* *.whl
 
-WORKDIR /opt/torch_tensorrt
+# WORKDIR /opt/torch_tensorrt
 
-ENV LD_LIBRARY_PATH /opt/python3/site-packages/torch/lib:/opt/python3/site-packages/torch_tensorrt/lib:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
-ENV PATH /opt/python3/site-packages/torch_tensorrt/bin:${PATH}
+# ENV LD_LIBRARY_PATH /opt/python3/site-packages/torch/lib:/opt/python3/site-packages/torch_tensorrt/lib:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
+# ENV PATH /opt/python3/site-packages/torch_tensorrt/bin:${PATH}
 
 CMD /bin/bash
diff --git a/docker/dist-build.sh b/docker/dist-build.sh
@@ -10,8 +10,9 @@ fi
 
 cd ${TOP_DIR} \
     && mkdir -p dist && cd py \
-    && pip install -r requirements.txt \
-    && pip install wheel
+    && pip install wheel \
+    && apt-get clean \
+    && pip install -r requirements.txt
 
 # Build Torch-TRT
 MAX_JOBS=1 LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 ${BUILD_CMD} $* || exit 1