From 44438a6514cf5facc8eec8c1be8c625c98cccae9 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Tue, 6 Feb 2024 17:08:18 -0600 Subject: [PATCH 1/2] Migrate CMSRucioClient base image to almalinux9 Build script for CMSRucioClient image --- docker/CMSRucioClient/Dockerfile | 26 +++++--- docker/CMSRucioClient/Dockerfile.centos7 | 82 ++++++++++++++++++++++++ docker/CMSRucioClient/build.sh | 10 +++ 3 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 docker/CMSRucioClient/Dockerfile.centos7 create mode 100755 docker/CMSRucioClient/build.sh diff --git a/docker/CMSRucioClient/Dockerfile b/docker/CMSRucioClient/Dockerfile index 4831e548..a7907d05 100644 --- a/docker/CMSRucioClient/Dockerfile +++ b/docker/CMSRucioClient/Dockerfile @@ -1,4 +1,6 @@ -FROM centos:7 +FROM almalinux:9 + +ARG RUCIO_VERSION RUN yum upgrade -y && \ yum clean all && \ @@ -8,10 +10,16 @@ RUN yum install -y epel-release.noarch && \ yum clean all && \ rm -rf /var/cache/yum +RUN yum install -y yum-utils && \ + yum clean all && \ + rm -rf /var/cache/yum + +RUN yum-config-manager --set-enabled crb + # FIXME: What is all this? We can't need it all RUN yum install -y python3 python3-pip pylint \ - httpd mod_wsgi libaio gcc python-devel.x86_64 \ - install wget unzip \ + httpd mod_wsgi libaio gcc python3-devel.x86_64 \ + wget unzip \ mod_ssl openssl-devel.x86_64 bind-utils nano git && \ yum clean all && \ rm -rf /var/cache/yum @@ -23,7 +31,7 @@ RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/ruc # Install WLCG repo for CMS VOMS -RUN yum -y install http://linuxsoft.cern.ch/wlcg/centos7/x86_64/wlcg-repo-1.0.0-1.el7.noarch.rpm && \ +RUN yum -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ yum clean all && \ rm -rf /var/cache/yum @@ -32,8 +40,8 @@ RUN yum -y install http://linuxsoft.cern.ch/wlcg/centos7/x86_64/wlcg-repo-1.0.0- RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true \ && yum install -y which krb5-devel gridsite \ - lcg-CA globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ - gfal2-python gfal2-all gfal2-util gfal2-python3 \ + globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ + gfal2-all gfal2-util python3-gfal2 \ nordugrid-arc-client nordugrid-arc-plugins-gfal \ nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \ nordugrid-arc-plugins-xrootd \ @@ -46,7 +54,7 @@ RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true \ RUN pip3 install --upgrade pip RUN pip3 install --upgrade setuptools -RUN pip3 install rucio-clients +RUN pip3 -v install rucio-clients==$RUCIO_VERSION RUN pip3 install python-gitlab RUN pip3 install liboidcagent @@ -75,8 +83,8 @@ RUN mkdir -p /etc/grid-security ; ln -s /cvmfs/grid.cern.ch/etc/grid-security/ce # COPY scripts /root/CMSRucio/docker/CMSRucioClient/scripts # Our schema -ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python2.7/site-packages/rucio/common/schema/cms.py -RUN python -m compileall /usr/lib/python2.7/site-packages/rucio/common/schema +ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python3.9/site-packages/rucio/common/schema/cms.py +RUN python3 -m compileall /usr/lib/python3.9/site-packages/rucio/common/schema ENV X509_USER_PROXY=/tmp/x509up ENTRYPOINT ["/bin/bash", "-c", "sleep infinity"] diff --git a/docker/CMSRucioClient/Dockerfile.centos7 b/docker/CMSRucioClient/Dockerfile.centos7 new file mode 100644 index 00000000..08958f6c --- /dev/null +++ b/docker/CMSRucioClient/Dockerfile.centos7 @@ -0,0 +1,82 @@ +FROM centos:7 + +RUN yum upgrade -y && \ + yum clean all && \ + rm -rf /var/cache/yum + +RUN yum install -y epel-release.noarch && \ + yum clean all && \ + rm -rf /var/cache/yum + +# FIXME: What is all this? We can't need it all +RUN yum install -y python3 python3-pip pylint \ + httpd mod_wsgi libaio gcc python-devel.x86_64 \ + install wget unzip \ + mod_ssl openssl-devel.x86_64 bind-utils nano git && \ + yum clean all && \ + rm -rf /var/cache/yum + +WORKDIR /tmp + +# Get various repos for yum setup +RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/rucio/1.20.3/etc/docker/dev/ca.repo + +# Install WLCG repo for CMS VOMS + +RUN yum -y install http://linuxsoft.cern.ch/wlcg/centos7/x86_64/wlcg-repo-1.0.0-1.el7.noarch.rpm && \ + yum clean all && \ + rm -rf /var/cache/yum + +# Install sets of RPM +# FIXME: This overlaps with above + +RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true \ + && yum install -y which krb5-devel gridsite \ + lcg-CA globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ + gfal2-python gfal2-all gfal2-util gfal2-python3 \ + nordugrid-arc-client nordugrid-arc-plugins-gfal \ + nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \ + nordugrid-arc-plugins-xrootd \ + xrootd-client && \ + yum clean all && \ + rm -rf /var/cache/yum + + +# Install Rucio and other stuff + +RUN pip3 install --upgrade pip +RUN pip3 install --upgrade setuptools +RUN pip3 -v install rucio-clients +RUN pip3 install python-gitlab +RUN pip3 install liboidcagent + +# Install DAS client to pull info from PhEDEx/DBS +RUN curl -k -o /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient +RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt + +WORKDIR /root +RUN git clone https://github.com/rucio/rucio.git + +COPY scripts /scripts +COPY loadtest /loadtest + +COPY rucio-testbed.cfg /opt/rucio-testbed/etc/rucio.cfg +COPY rucio-dev.cfg /opt/rucio-dev/etc/rucio.cfg +COPY rucio-int.cfg /opt/rucio-int/etc/rucio.cfg +COPY rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg + +# Pull some useful stuff out of git +WORKDIR /root +RUN git clone https://github.com/dmwm/CMSRucio.git + +RUN mkdir -p /etc/grid-security ; ln -s /cvmfs/grid.cern.ch/etc/grid-security/certificates /etc/grid-security/ + +# Overwrite what's there for testing +# COPY scripts /root/CMSRucio/docker/CMSRucioClient/scripts + +# Our schema +ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python2.7/site-packages/rucio/common/schema/cms.py +RUN python -m compileall /usr/lib/python2.7/site-packages/rucio/common/schema + +ENV X509_USER_PROXY=/tmp/x509up +ENTRYPOINT ["/bin/bash", "-c", "sleep infinity"] diff --git a/docker/CMSRucioClient/build.sh b/docker/CMSRucioClient/build.sh new file mode 100755 index 00000000..0f320b26 --- /dev/null +++ b/docker/CMSRucioClient/build.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +export RUCIO_VERSION=32.3.1 +export CMS_VERSION=${RUCIO_VERSION}.cms1 + +export HARBOR=registry.cern.ch/cmsrucio + +podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f Dockerfile -t $HARBOR/rucio-client:release-$CMS_VERSION . \ No newline at end of file From 07376f478b1553dc91a7bd0b96d501c3b094c064 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Wed, 7 Feb 2024 15:42:11 -0600 Subject: [PATCH 2/2] Use dnf instead of yum in CMSRucioClient --- docker/CMSRucioClient/Dockerfile | 45 ++++++++++++++------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/docker/CMSRucioClient/Dockerfile b/docker/CMSRucioClient/Dockerfile index a7907d05..715ad2d2 100644 --- a/docker/CMSRucioClient/Dockerfile +++ b/docker/CMSRucioClient/Dockerfile @@ -2,27 +2,23 @@ FROM almalinux:9 ARG RUCIO_VERSION -RUN yum upgrade -y && \ - yum clean all && \ - rm -rf /var/cache/yum +RUN dnf upgrade -y && \ + dnf clean all && \ + rm -rf /var/cache/dnf -RUN yum install -y epel-release.noarch && \ - yum clean all && \ - rm -rf /var/cache/yum +RUN dnf install -y epel-release.noarch && \ + dnf clean all && \ + rm -rf /var/cache/dnf -RUN yum install -y yum-utils && \ - yum clean all && \ - rm -rf /var/cache/yum - -RUN yum-config-manager --set-enabled crb +RUN dnf config-manager --set-enabled crb # FIXME: What is all this? We can't need it all -RUN yum install -y python3 python3-pip pylint \ - httpd mod_wsgi libaio gcc python3-devel.x86_64 \ +RUN dnf install -y python3 python3-pip pylint \ + httpd mod_wsgi libaio gcc python3-devel \ wget unzip \ - mod_ssl openssl-devel.x86_64 bind-utils nano git && \ - yum clean all && \ - rm -rf /var/cache/yum + mod_ssl openssl-devel bind-utils nano git && \ + dnf clean all && \ + rm -rf /var/cache/dnf WORKDIR /tmp @@ -30,24 +26,23 @@ WORKDIR /tmp RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/rucio/1.20.3/etc/docker/dev/ca.repo # Install WLCG repo for CMS VOMS - -RUN yum -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ - yum clean all && \ - rm -rf /var/cache/yum +RUN dnf -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm && \ + dnf clean all && \ + rm -rf /var/cache/dnf # Install sets of RPM # FIXME: This overlaps with above -RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true \ - && yum install -y which krb5-devel gridsite \ +RUN dnf config-manager --save --setopt=wlcg.skip_if_unavailable=true \ + && dnf install -y which krb5-devel gridsite \ globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ - gfal2-all gfal2-util python3-gfal2 \ + gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 \ nordugrid-arc-client nordugrid-arc-plugins-gfal \ nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \ nordugrid-arc-plugins-xrootd \ xrootd-client && \ - yum clean all && \ - rm -rf /var/cache/yum + dnf clean all && \ + rm -rf /var/cache/dnf # Install Rucio and other stuff