From e1d9e26ca88477c5e27f90f96d5a8795f218a34e Mon Sep 17 00:00:00 2001 From: Ian Harry Date: Fri, 26 Aug 2022 09:51:08 +0100 Subject: [PATCH] Update Docker and venv to python3.9 (#4115) * Update Docker files * Also update venv * Also thie file * Probably don't want python38-devel for python39 --- .github/workflows/build_venv.yml | 2 +- .github/workflows/docker-build.yml | 2 +- Dockerfile | 8 ++++---- docker/etc/docker-install.sh | 8 +++++--- tools/docker_build_dist.sh | 4 ++-- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_venv.yml b/.github/workflows/build_venv.yml index d68ed8ca7fb..cf7e49235ce 100644 --- a/.github/workflows/build_venv.yml +++ b/.github/workflows/build_venv.yml @@ -12,7 +12,7 @@ jobs: name: "Set up Python" uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - env: OSG_ACCESS: "${{secrets.OSG_ACCESS}}" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5048d13a5ae..235f518015f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,7 +12,7 @@ jobs: name: "Set up Python" uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: "Preparing a host container" run: "docker build -t pycbc-docker-tmp ." diff --git a/Dockerfile b/Dockerfile index 4f1a2db2a6f..7002c97072a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ADD docker/etc/cvmfs/config-osg.opensciencegrid.org.conf /etc/cvmfs/config-osg.o RUN dnf -y install https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm && dnf -y install cvmfs cvmfs-config-default && dnf clean all && dnf makecache && \ dnf -y groupinstall "Development Tools" \ "Scientific Support" && \ - rpm -e --nodeps git perl-Git && dnf -y install @python38 rsync zlib-devel libpng-devel libjpeg-devel sqlite-devel openssl-devel fftw-libs-single fftw-devel fftw fftw-libs-long fftw-libs fftw-libs-double gsl gsl-devel hdf5 hdf5-devel python38-devel swig which osg-ca-certs && python3.8 -m pip install --upgrade pip setuptools wheel cython && python3.8 -m pip install mkl ipython jupyter jupyterhub jupyterlab lalsuite && \ + rpm -e --nodeps git perl-Git && dnf -y install @python39 rsync zlib-devel libpng-devel libjpeg-devel sqlite-devel openssl-devel fftw-libs-single fftw-devel fftw fftw-libs-long fftw-libs fftw-libs-double gsl gsl-devel hdf5 hdf5-devel python39-devel swig which osg-ca-certs && python3.9 -m pip install --upgrade pip setuptools wheel cython && python3.9 -m pip install mkl ipython jupyter jupyterhub jupyterlab lalsuite && \ dnf -y install https://repo.opensciencegrid.org/osg/3.5/el8/testing/x86_64/osg-wn-client-3.5-5.osg35.el8.noarch.rpm && dnf clean all # set up environment @@ -21,15 +21,15 @@ RUN cd / && \ # Install MPI software needed for pycbc_inference # at the end. RUN dnf -y install libibverbs libibverbs-devel libibmad libibmad-devel libibumad libibumad-devel librdmacm librdmacm-devel libmlx5 libmlx4 openmpi openmpi-devel && \ - python3.8 -m pip install schwimmbad && \ - MPICC=/lib64/openmpi/bin/mpicc CFLAGS='-I /usr/include/openmpi-x86_64/ -L /usr/lib64/openmpi/lib/ -lmpi' python3.8 -m pip install --no-cache-dir mpi4py + python3.9 -m pip install schwimmbad && \ + MPICC=/lib64/openmpi/bin/mpicc CFLAGS='-I /usr/include/openmpi-x86_64/ -L /usr/lib64/openmpi/lib/ -lmpi' python3.9 -m pip install --no-cache-dir mpi4py RUN echo "/usr/lib64/openmpi/lib/" > /etc/ld.so.conf.d/openmpi.conf # Now update all of our library installations RUN rm -f /etc/ld.so.cache && /sbin/ldconfig # Make python be what we want -RUN alternatives --set python /usr/bin/python3.8 +RUN alternatives --set python /usr/bin/python3.9 # Explicitly set the path so that it is not inherited from build the environment ENV PATH "/usr/local/bin:/usr/bin:/bin:/lib64/openmpi/bin/bin" diff --git a/docker/etc/docker-install.sh b/docker/etc/docker-install.sh index fdaa49e74b4..68d31814e96 100755 --- a/docker/etc/docker-install.sh +++ b/docker/etc/docker-install.sh @@ -1,9 +1,11 @@ #!/bin/bash -v set -e cd /scratch -python3.8 -m pip install --upgrade 'pip<22.0' -python3.8 -m pip install -r requirements.txt -python3.8 -m pip install . +python3.9 -m pip install --upgrade 'pip<22.0' +python3.9 -m pip install -r requirements.txt +python3.9 -m pip install -r requirements-igwn.txt +python3.9 -m pip install -r companion.txt +python3.9 -m pip install . cd / mkdir -p /opt/pycbc/src cp -a /scratch /opt/pycbc/src/pycbc diff --git a/tools/docker_build_dist.sh b/tools/docker_build_dist.sh index 7b40a76e3da..3a02d056f22 100755 --- a/tools/docker_build_dist.sh +++ b/tools/docker_build_dist.sh @@ -37,7 +37,7 @@ fi if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then ENV_OS="x86_64_rhel_8" - yum -y install python38 python38-devel + yum -y install python39 python39-devel yum -y groupinstall "Development Tools" yum -y install which rsync yum clean all @@ -51,7 +51,7 @@ if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then mkdir -p ${CVMFS_PATH} VENV_PATH=${CVMFS_PATH}/pycbc-${SOURCE_TAG} - virtualenv -p python3.8 ${VENV_PATH} + virtualenv -p python3.9 ${VENV_PATH} echo 'export PYTHONUSERBASE=${VIRTUAL_ENV}/.local' >> ${VENV_PATH}/bin/activate echo "export XDG_CACHE_HOME=\${HOME}/cvmfs-pycbc-${SOURCE_TAG}/.cache" >> ${VENV_PATH}/bin/activate source ${VENV_PATH}/bin/activate