Use tmp cache dirs (by default) on porter, dont override user settings (

#920) co-authored by @matthiasdiener
illinois-ceesd · Jun 28, 2023 · dd58da4 · dd58da4
1 parent 11472b4
commit dd58da4
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 3 deletions.
diff --git a/scripts/lassen-parallel-spawner.sh b/scripts/lassen-parallel-spawner.sh
@@ -4,7 +4,9 @@
 # unset CUDA_CACHE_DISABLE
 POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"}
 XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"}
-export POCL_CACHE_DIR="${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"
-export XDG_CACHE_HOME="${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"
+POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"}
+XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"}
+export POCL_CACHE_DIR
+export XDG_CACHE_HOME
 
 "$@"
diff --git a/scripts/mirge-testing-env.sh b/scripts/mirge-testing-env.sh
@@ -21,8 +21,9 @@ MIRGE_MPI_EXEC="mpiexec"
 PYOPENCL_TEST=""
 PYOPENCL_CTX=""
 
+# Add new hosts here, and <hostname>-parallel-spawner.sh
 if [[ $(hostname) == "porter" ]]; then
-    MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/run-gpus-generic.sh"
+    MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/porter-parallel-spawner.sh"
     PYOPENCL_TEST="port:nv"
     PYOPENCL_CTX="port:nv"
 

diff --git a/scripts/run-gpus-generic.sh b/scripts/run-gpus-generic.sh
@@ -11,13 +11,24 @@
 #
 # Run it like this:
 #   mpiexec -n 2 bash run-gpus-generic.sh python -m mpi4py pulse-mpi.py --lazy
+# unset CUDA_CACHE_DISABLE
+POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"}
+XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"}
 
 if [[ -n "$OMPI_COMM_WORLD_NODE_RANK" ]]; then
     # Open MPI
     export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
+    RANK_ID="rank${OMPI_COMM_WORLD_RANK}"
 elif [[ -n "$MPI_LOCALRANKID" ]]; then
     # mpich/mvapich
     export CUDA_VISIBLE_DEVICES=$MPI_LOCALRANKID
+    RANK_ID="rank${MPI_LOCALRANKID}"
 fi
 
+POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/${RANK_ID}"}
+XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/${RANK_ID}"}
+
+export POCL_CACHE_DIR
+export XDG_CACHE_HOME
+
 "$@"