diff --git a/ci/htcondor.sh b/ci/htcondor.sh index ec2c8417..f5dcb8a8 100755 --- a/ci/htcondor.sh +++ b/ci/htcondor.sh @@ -21,13 +21,13 @@ function jobqueue_before_install { function jobqueue_install { cd ./ci/htcondor - docker-compose exec -T submit /bin/bash -c "cd /dask-jobqueue; pip3 install -e .;chown -R submituser ." + docker-compose exec -T submit conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip3 install -e .;chown -R submituser ." cd - } function jobqueue_script { cd ./ci/htcondor - docker-compose exec -T --user submituser submit /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --log-cli-level DEBUG --capture=tee-sys --verbose -E htcondor " + docker-compose exec -T --user submituser submit conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --log-cli-level DEBUG --capture=tee-sys --verbose -E htcondor " cd - } diff --git a/ci/htcondor/Dockerfile b/ci/htcondor/Dockerfile index 06e0d708..2efaf0d7 100644 --- a/ci/htcondor/Dockerfile +++ b/ci/htcondor/Dockerfile @@ -7,7 +7,8 @@ RUN curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-L ENV PATH /opt/anaconda/bin:$PATH # environment.yml file is copied by CI script. If manually building, you should copy it too from parent directory COPY environment.yml . -RUN conda env update -n base --file environment.yml +RUN conda env create --file environment.yml +SHELL ["conda", "run", "-n", "dask-jobqueue", "/bin/bash", "-c"] FROM htcondor/execute:el7 as execute diff --git a/ci/pbs.sh b/ci/pbs.sh index b4af1127..daa9a712 100644 --- a/ci/pbs.sh +++ b/ci/pbs.sh @@ -20,11 +20,11 @@ function jobqueue_before_install { } function jobqueue_install { - docker exec pbs_master /bin/bash -c "cd /dask-jobqueue; pip install -e .; chown -R pbsuser ." + docker exec pbs_master conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip install -e .; chown -R pbsuser ." } function jobqueue_script { - docker exec -u pbsuser pbs_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" + docker exec -u pbsuser pbs_master conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E pbs" } function jobqueue_after_script { diff --git a/ci/pbs/Dockerfile b/ci/pbs/Dockerfile index bb338a0a..a13d9002 100644 --- a/ci/pbs/Dockerfile +++ b/ci/pbs/Dockerfile @@ -32,7 +32,8 @@ RUN curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-L rm -f miniconda.sh # environment.yml file is copied by CI script. If manually building, you should copy it too from parent directory COPY environment.yml . -RUN conda env update -n base --file environment.yml +RUN conda env create --file environment.yml +SHELL ["conda", "run", "-n", "dask-jobqueue", "/bin/bash", "-c"] # Copy entrypoint and other needed scripts COPY ./*.sh / diff --git a/ci/sge.sh b/ci/sge.sh index 227e8db2..fd08dca3 100644 --- a/ci/sge.sh +++ b/ci/sge.sh @@ -19,11 +19,11 @@ function jobqueue_before_install { } function jobqueue_install { - docker exec sge_master /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec sge_master conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec sge_master /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -s -E sge" + docker exec sge_master conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose --full-trace -s -E sge" } function jobqueue_after_script { diff --git a/ci/sge/Dockerfile b/ci/sge/Dockerfile index dfc60a2e..5c60da26 100644 --- a/ci/sge/Dockerfile +++ b/ci/sge/Dockerfile @@ -11,7 +11,8 @@ RUN curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-L ENV PATH /opt/anaconda/bin:$PATH # environment.yml file is copied by CI script. If manually building, you should copy it too from parent directory COPY environment.yml . -RUN conda env update -n base --file environment.yml +RUN conda env create --file environment.yml +SHELL ["conda", "run", "-n", "dask-jobqueue", "/bin/bash", "-c"] COPY ./*.sh / COPY ./*.txt / diff --git a/ci/sge/run-master.sh b/ci/sge/run-master.sh index 6b42dc66..2157a8d0 100755 --- a/ci/sge/run-master.sh +++ b/ci/sge/run-master.sh @@ -9,13 +9,12 @@ while ! ping -c1 slave_one &>/dev/null; do :; done while ! qconf -sconf &>/dev/null; do sleep 0.1; done cat /var/lib/gridengine//default/common/act_qmaster -qconf -Msconf /scheduler.txt -qconf -Ahgrp /hosts.txt -qconf -Aq /queue.txt +qconf -Msconf /dask-jobqueue/ci/sge/scheduler.txt +qconf -Ahgrp /dask-jobqueue/ci/sge/hosts.txt +qconf -Aq /dask-jobqueue/ci/sge/queue.txt qconf -ah slave_one qconf -ah slave_two -qconf -ah slave_three qconf -as $HOSTNAME bash add_worker.sh dask.q slave_one 4 diff --git a/ci/slurm.sh b/ci/slurm.sh index 9e1fa2ba..10acc847 100644 --- a/ci/slurm.sh +++ b/ci/slurm.sh @@ -22,17 +22,17 @@ function show_network_interfaces { for c in slurmctld c1 c2; do echo '------------------------------------------------------------' echo docker container: $c - docker exec $c python -c 'import psutil; print(psutil.net_if_addrs().keys())' + docker exec $c conda run -n dask-jobqueue python -c 'import psutil; print(psutil.net_if_addrs().keys())' echo '------------------------------------------------------------' done } function jobqueue_install { - docker exec slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ." + docker exec slurmctld conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip install -e ." } function jobqueue_script { - docker exec slurmctld /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" + docker exec slurmctld conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s" } function jobqueue_after_script { diff --git a/ci/slurm/Dockerfile b/ci/slurm/Dockerfile index 0eaeaead..4415681a 100644 --- a/ci/slurm/Dockerfile +++ b/ci/slurm/Dockerfile @@ -9,7 +9,8 @@ RUN curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-L ENV PATH /opt/anaconda/bin:$PATH # environment.yml file is copied by CI script. If manually building, you should copy it too from parent directory COPY environment.yml . -RUN conda env update -n base --file environment.yml +RUN conda env create --file environment.yml +SHELL ["conda", "run", "-n", "dask-jobqueue", "/bin/bash", "-c"] ENV LC_ALL en_US.UTF-8