diff --git a/requirements.txt b/requirements.txt index c01bb4a1f..aa8aed495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,14 +12,14 @@ git+https://github.com/pythological/kanren.git#egg=miniKanren # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic #--editable git+https://github.com/inducer/pyopencl.git#egg=pyopencl ---editable git+https://github.com/kaushikcfd/loopy.git#egg=loopy +--editable git+https://github.com/illinois-ceesd/loopy.git@production#egg=loopy --editable git+https://github.com/inducer/dagrt.git#egg=dagrt --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy ---editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext ---editable git+https://github.com/majosm/meshmode.git@production#egg=meshmode ---editable git+https://github.com/majosm/grudge.git@coupled-volumes#egg=grudge ---editable git+https://github.com/kaushikcfd/pytato.git#egg=pytato +--editable git+https://github.com/illinois-ceesd/arraycontext.git@production#egg=arraycontext +--editable git+https://github.com/illinois-ceesd/meshmode.git@production#egg=meshmode +--editable git+https://github.com/illinois-ceesd/grudge.git@production#egg=grudge +--editable git+https://github.com/illinois-ceesd/pytato.git@production#egg=pytato --editable git+https://github.com/pyrometheus/pyrometheus.git@tulio-transport#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle --editable git+https://github.com/kaushikcfd/feinsum.git#egg=feinsum diff --git a/scripts/lassen-parallel-spawner.sh b/scripts/lassen-parallel-spawner.sh index 70ab0847e..75efa4668 100755 --- a/scripts/lassen-parallel-spawner.sh +++ b/scripts/lassen-parallel-spawner.sh @@ -4,7 +4,9 @@ # unset CUDA_CACHE_DISABLE POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"} XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"} -export POCL_CACHE_DIR="${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK" -export XDG_CACHE_HOME="${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK" +POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"} +XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/rank$OMPI_COMM_WORLD_RANK"} +export POCL_CACHE_DIR +export XDG_CACHE_HOME "$@" diff --git a/scripts/mirge-testing-env.sh b/scripts/mirge-testing-env.sh index baf8512aa..017f30ed1 100755 --- a/scripts/mirge-testing-env.sh +++ b/scripts/mirge-testing-env.sh @@ -21,8 +21,9 @@ MIRGE_MPI_EXEC="mpiexec" PYOPENCL_TEST="" PYOPENCL_CTX="" +# Add new hosts here, and -parallel-spawner.sh if [[ $(hostname) == "porter" ]]; then - MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/run-gpus-generic.sh" + MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/porter-parallel-spawner.sh" PYOPENCL_TEST="port:nv" PYOPENCL_CTX="port:nv" diff --git a/scripts/run-gpus-generic.sh b/scripts/run-gpus-generic.sh index 39763f99b..cec85c8f7 100755 --- a/scripts/run-gpus-generic.sh +++ b/scripts/run-gpus-generic.sh @@ -11,13 +11,24 @@ # # Run it like this: # mpiexec -n 2 bash run-gpus-generic.sh python -m mpi4py pulse-mpi.py --lazy +# unset CUDA_CACHE_DISABLE +POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"} +XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"} if [[ -n "$OMPI_COMM_WORLD_NODE_RANK" ]]; then # Open MPI export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK + RANK_ID="rank${OMPI_COMM_WORLD_RANK}" elif [[ -n "$MPI_LOCALRANKID" ]]; then # mpich/mvapich export CUDA_VISIBLE_DEVICES=$MPI_LOCALRANKID + RANK_ID="rank${MPI_LOCALRANKID}" fi +POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/${RANK_ID}"} +XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/${RANK_ID}"} + +export POCL_CACHE_DIR +export XDG_CACHE_HOME + "$@"