ML: install IPEX and oneCCL binding using pip instead of conda (oap-p…

…roject#1278)
kiwik · Mar 28, 2023 · b2f92a0 · b2f92a0
1 parent b4b3dff
commit b2f92a0
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 9 deletions.
diff --git a/docker/runtime/ml/oneapi/Dockerfile b/docker/runtime/ml/oneapi/Dockerfile
@@ -30,12 +30,14 @@ RUN export PATH="$HOME/anaconda3/envs/$CLOUDTIK_ENV/bin:$PATH" \
     && pip --no-cache-dir install -r /tmp/requirements.txt \
     && sudo rm /tmp/requirements.txt \
     # Install deep learning libraries (PyTorch and torch tools)
-    && conda install -c intel pytorch=1.13.0 torchvision=0.14.1 \
-        intel-extension-for-pytorch=1.13.10 oneccl_bind_pt=1.13.0 \
-        -p $HOME/anaconda3/envs/$CLOUDTIK_ENV -y \
-    # Install Jemalloc for better performance
+    && pip --no-cache-dir install torch==1.13.1 torchvision==0.14.1 \
+        --extra-index-url https://download.pytorch.org/whl/cpu \
+    # Install Jemalloc for better performance for Intel Extension for PyTorch
     && conda install jemalloc -p $HOME/anaconda3/envs/$CLOUDTIK_ENV -y \
     && conda clean -itqy \
+    # Install Intel Extension for PyTorch and oneCCL binding
+    && pip --no-cache-dir install intel-extension-for-pytorch==1.13.100+cpu \
+        oneccl_bind_pt==1.13.0+cpu -f https://developer.intel.com/ipex-whl-stable-cpu \
     # Source the mpi and ccl vars
     && . /opt/intel/oneapi/mpi/latest/env/vars.sh \
     && . /opt/intel/oneapi/tbb/latest/env/vars.sh \

diff --git a/python/cloudtik/runtime/ml/scripts/install.sh b/python/cloudtik/runtime/ml/scripts/install.sh
@@ -68,16 +68,15 @@ function install_ml() {
     else
         echo "Installing deep learning frameworks: tensorflow, pytorch..."
         pip --no-cache-dir -qq install tensorflow==2.9.3
+        pip --no-cache-dir -qq install torch==1.13.1 torchvision==0.14.1 \
+            --extra-index-url https://download.pytorch.org/whl/cpu
 
         if ([ "$ML_WITH_ONEAPI" == "true" ] || [ "$ML_WITH_INTEL_PYTORCH" == "true" ]) \
             && [ "$ML_WITH_INTEL_PYTORCH" != "false" ]; then
-            conda install -q pytorch=1.13.0 torchvision=0.14.1 \
-              oneccl_bind_pt=1.13.0 intel-extension-for-pytorch=1.13.10 -p ${CLOUDTIK_ENV_ROOT} -c intel -y > /dev/null
             # Install Jemalloc for better performance
             conda install jemalloc -p ${CLOUDTIK_ENV_ROOT} -y > /dev/null
-        else
-            pip --no-cache-dir -qq install torch==1.13.1 torchvision==0.14.1 \
-              --extra-index-url https://download.pytorch.org/whl/cpu
+            pip --no-cache-dir -qq install intel-extension-for-pytorch==1.13.100+cpu \
+                oneccl_bind_pt==1.13.0+cpu -f https://developer.intel.com/ipex-whl-stable-cpu
         fi
         pip --no-cache-dir -qq install transformers==4.11.0
     fi