From dd58da4dc26fb5c8ed70e5d6849649ecd8b66532 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Jun 2023 19:46:44 -0500 Subject: [PATCH 1/2] Use tmp cache dirs (by default) on porter, dont override user settings (#920) co-authored by @matthiasdiener --- scripts/lassen-parallel-spawner.sh | 6 ++++-- scripts/mirge-testing-env.sh | 3 ++- scripts/run-gpus-generic.sh | 11 +++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/lassen-parallel-spawner.sh b/scripts/lassen-parallel-spawner.sh index 70ab0847e..75efa4668 100755 --- a/scripts/lassen-parallel-spawner.sh +++ b/scripts/lassen-parallel-spawner.sh @@ -4,7 +4,9 @@ # unset CUDA_CACHE_DISABLE POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"} XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"} -export POCL_CACHE_DIR="${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK" -export XDG_CACHE_HOME="${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK" +POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"} +XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"} +export POCL_CACHE_DIR +export XDG_CACHE_HOME "$@" diff --git a/scripts/mirge-testing-env.sh b/scripts/mirge-testing-env.sh index baf8512aa..017f30ed1 100755 --- a/scripts/mirge-testing-env.sh +++ b/scripts/mirge-testing-env.sh @@ -21,8 +21,9 @@ MIRGE_MPI_EXEC="mpiexec" PYOPENCL_TEST="" PYOPENCL_CTX="" +# Add new hosts here, and -parallel-spawner.sh if [[ $(hostname) == "porter" ]]; then - MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/run-gpus-generic.sh" + MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/porter-parallel-spawner.sh" PYOPENCL_TEST="port:nv" PYOPENCL_CTX="port:nv" diff --git a/scripts/run-gpus-generic.sh b/scripts/run-gpus-generic.sh index 39763f99b..cec85c8f7 100755 --- a/scripts/run-gpus-generic.sh +++ b/scripts/run-gpus-generic.sh @@ -11,13 +11,24 @@ # # Run it like this: # mpiexec -n 2 bash run-gpus-generic.sh python -m mpi4py pulse-mpi.py --lazy +# unset CUDA_CACHE_DISABLE +POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"} +XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"} if [[ -n "$OMPI_COMM_WORLD_NODE_RANK" ]]; then # Open MPI export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK + RANK_ID="rank${OMPI_COMM_WORLD_RANK}" elif [[ -n "$MPI_LOCALRANKID" ]]; then # mpich/mvapich export CUDA_VISIBLE_DEVICES=$MPI_LOCALRANKID + RANK_ID="rank${MPI_LOCALRANKID}" fi +POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/${RANK_ID}"} +XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/${RANK_ID}"} + +export POCL_CACHE_DIR +export XDG_CACHE_HOME + "$@" From 0952c0f5bb41ff47ef09d907739da9029e9d3f3b Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Jun 2023 01:33:10 -0500 Subject: [PATCH 2/2] Use illinois-ceesd for upstream (#919) --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index c01bb4a1f..aa8aed495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,14 +12,14 @@ git+https://github.com/pythological/kanren.git#egg=miniKanren # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic #--editable git+https://github.com/inducer/pyopencl.git#egg=pyopencl ---editable git+https://github.com/kaushikcfd/loopy.git#egg=loopy +--editable git+https://github.com/illinois-ceesd/loopy.git@production#egg=loopy --editable git+https://github.com/inducer/dagrt.git#egg=dagrt --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy ---editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext ---editable git+https://github.com/majosm/meshmode.git@production#egg=meshmode ---editable git+https://github.com/majosm/grudge.git@coupled-volumes#egg=grudge ---editable git+https://github.com/kaushikcfd/pytato.git#egg=pytato +--editable git+https://github.com/illinois-ceesd/arraycontext.git@production#egg=arraycontext +--editable git+https://github.com/illinois-ceesd/meshmode.git@production#egg=meshmode +--editable git+https://github.com/illinois-ceesd/grudge.git@production#egg=grudge +--editable git+https://github.com/illinois-ceesd/pytato.git@production#egg=pytato --editable git+https://github.com/pyrometheus/pyrometheus.git@tulio-transport#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle --editable git+https://github.com/kaushikcfd/feinsum.git#egg=feinsum