-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Templatize Dockerfiles & update workflows (#223)
Now build images by a script with a shared Dockerfile template --------- Co-authored-by: Binyang Li <[email protected]> Co-authored-by: Saeed Maleki <[email protected]>
- Loading branch information
3 people
authored
Nov 22, 2023
1 parent
15f6dcc
commit dab19e0
Showing
16 changed files
with
113 additions
and
222 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| ARG BASE_IMAGE=ghcr.io/microsoft/mscclpp/mscclpp:base-cuda12.1 | ||
| FROM ${BASE_IMAGE} | ||
|
|
||
| LABEL maintainer="MSCCL++" | ||
| LABEL org.opencontainers.image.source https://github.com/microsoft/mscclpp | ||
|
|
||
| RUN apt-get update && \ | ||
| apt-get install -y --no-install-recommends \ | ||
| htop \ | ||
| lcov \ | ||
| vim \ | ||
| && \ | ||
| apt-get autoremove && \ | ||
| apt-get clean && \ | ||
| rm -rf /var/lib/apt/lists/* /tmp/* | ||
|
|
||
| # Install cmake 3.26.4 | ||
| ENV CMAKE_VERSION="3.26.4" | ||
| ENV CMAKE_HOME="/tmp/cmake-${CMAKE_VERSION}-linux-x86_64" \ | ||
| CMAKE_URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" | ||
| RUN curl -L ${CMAKE_URL} -o ${CMAKE_HOME}.tar.gz && \ | ||
| tar xzf ${CMAKE_HOME}.tar.gz -C /usr/local && \ | ||
| rm -rf ${CMAKE_HOME}.tar.gz | ||
| ENV PATH="/usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH}" | ||
|
|
||
| # Install Python dependencies | ||
| ADD . /tmp/mscclpp | ||
| WORKDIR /tmp/mscclpp | ||
| ARG TARGET="cuda12.1" | ||
| RUN cuda_major_version=$(echo ${TARGET} | grep -oP 'cuda\K[0-9]+') && \ | ||
| python3 -m pip install --no-cache-dir -r python/requirements_cu${cuda_major_version}.txt | ||
|
|
||
| # Set PATH | ||
| RUN echo PATH="${PATH}" > /etc/environment | ||
|
|
||
| # Cleanup | ||
| RUN rm -rf /tmp/mscclpp | ||
| WORKDIR / |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -e | ||
|
|
||
| declare -A baseImageTable | ||
| baseImageTable=( | ||
| ["cuda11.8"]="nvidia/cuda:11.8.0-devel-ubuntu20.04" | ||
| ["cuda12.1"]="nvidia/cuda:12.1.1-devel-ubuntu20.04" | ||
| ["cuda12.2"]="nvidia/cuda:12.2.2-devel-ubuntu20.04" | ||
| ) | ||
|
|
||
| declare -A extraLdPathTable | ||
| extraLdPathTable=( | ||
| ["cuda11.8"]="/usr/local/cuda-11.8/lib64" | ||
| ["cuda12.1"]="/usr/local/cuda-12.1/compat:/usr/local/cuda-12.1/lib64" | ||
| ["cuda12.2"]="/usr/local/cuda-12.2/compat:/usr/local/cuda-12.2/lib64" | ||
| ) | ||
|
|
||
| GHCR="ghcr.io/microsoft/mscclpp/mscclpp" | ||
| TARGET=${1} | ||
|
|
||
| print_usage() { | ||
| echo "Usage: $0 [cuda11.8|cuda12.1|cuda12.2]" | ||
| } | ||
|
|
||
| if [[ ! -v "baseImageTable[${TARGET}]" ]]; then | ||
| echo "Invalid target: ${TARGET}" | ||
| print_usage | ||
| exit 1 | ||
| fi | ||
| echo "Target: ${TARGET}" | ||
|
|
||
| SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" | ||
|
|
||
| cd ${SCRIPT_DIR}/.. | ||
|
|
||
| docker build -t ${GHCR}:base-${TARGET} \ | ||
| -f docker/base-x.dockerfile \ | ||
| --build-arg BASE_IMAGE=${baseImageTable[${TARGET}]} \ | ||
| --build-arg EXTRA_LD_PATH=${extraLdPathTable[${TARGET}]} \ | ||
| --build-arg TARGET=${TARGET} . | ||
|
|
||
| docker build -t ${GHCR}:base-dev-${TARGET} \ | ||
| -f docker/base-dev-x.dockerfile \ | ||
| --build-arg BASE_IMAGE=${GHCR}:base-${TARGET} \ | ||
| --build-arg TARGET=${TARGET} . |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.