Merge branch 'main' into actx-init

illinois-ceesd · Jun 28, 2023 · 0befca5 · 0befca5
2 parents 6578353 + 0952c0f
commit 0befca5
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 8 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -12,14 +12,14 @@ git+https://github.com/pythological/kanren.git#egg=miniKanren
 # The following packages will be git cloned by emirge:
 --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic
 #--editable git+https://github.com/inducer/pyopencl.git#egg=pyopencl
---editable git+https://github.com/kaushikcfd/loopy.git#egg=loopy
+--editable git+https://github.com/illinois-ceesd/loopy.git@production#egg=loopy
 --editable git+https://github.com/inducer/dagrt.git#egg=dagrt
 --editable git+https://github.com/inducer/leap.git#egg=leap
 --editable git+https://github.com/inducer/modepy.git#egg=modepy
---editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext
---editable git+https://github.com/majosm/meshmode.git@production#egg=meshmode
---editable git+https://github.com/majosm/grudge.git@coupled-volumes#egg=grudge
---editable git+https://github.com/kaushikcfd/pytato.git#egg=pytato
+--editable git+https://github.com/illinois-ceesd/arraycontext.git@production#egg=arraycontext
+--editable git+https://github.com/illinois-ceesd/meshmode.git@production#egg=meshmode
+--editable git+https://github.com/illinois-ceesd/grudge.git@production#egg=grudge
+--editable git+https://github.com/illinois-ceesd/pytato.git@production#egg=pytato
 --editable git+https://github.com/pyrometheus/pyrometheus.git@tulio-transport#egg=pyrometheus
 --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle
 --editable git+https://github.com/kaushikcfd/feinsum.git#egg=feinsum

diff --git a/scripts/lassen-parallel-spawner.sh b/scripts/lassen-parallel-spawner.sh
@@ -4,7 +4,9 @@
 # unset CUDA_CACHE_DISABLE
 POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"}
 XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"}
-export POCL_CACHE_DIR="${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"
-export XDG_CACHE_HOME="${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"
+POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"}
+XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"}
+export POCL_CACHE_DIR
+export XDG_CACHE_HOME
 
 "$@"
diff --git a/scripts/mirge-testing-env.sh b/scripts/mirge-testing-env.sh
@@ -21,8 +21,9 @@ MIRGE_MPI_EXEC="mpiexec"
 PYOPENCL_TEST=""
 PYOPENCL_CTX=""
 
+# Add new hosts here, and <hostname>-parallel-spawner.sh
 if [[ $(hostname) == "porter" ]]; then
-    MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/run-gpus-generic.sh"
+    MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/porter-parallel-spawner.sh"
     PYOPENCL_TEST="port:nv"
     PYOPENCL_CTX="port:nv"
 

diff --git a/scripts/run-gpus-generic.sh b/scripts/run-gpus-generic.sh
@@ -11,13 +11,24 @@
 #
 # Run it like this:
 #   mpiexec -n 2 bash run-gpus-generic.sh python -m mpi4py pulse-mpi.py --lazy
+# unset CUDA_CACHE_DISABLE
+POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"}
+XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"}
 
 if [[ -n "$OMPI_COMM_WORLD_NODE_RANK" ]]; then
     # Open MPI
     export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
+    RANK_ID="rank${OMPI_COMM_WORLD_RANK}"
 elif [[ -n "$MPI_LOCALRANKID" ]]; then
     # mpich/mvapich
     export CUDA_VISIBLE_DEVICES=$MPI_LOCALRANKID
+    RANK_ID="rank${MPI_LOCALRANKID}"
 fi
 
+POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/${RANK_ID}"}
+XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/${RANK_ID}"}
+
+export POCL_CACHE_DIR
+export XDG_CACHE_HOME
+
 "$@"