From 3889a60c6da5ef0f60c4ec184d6a35643aecacfd Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Wed, 3 Apr 2024 15:16:07 -0500 Subject: [PATCH] Slimmed down rucio_client. Created rucio_client with alma9-minimal Updated rucio_client/README.md Use a requirements.txt file to manage script dependencies more clearly. --- docker/rucio_client/Dockerfile | 32 +++------- docker/rucio_client/Dockerfile.minimal | 66 ++++++++++++++++++++ docker/rucio_client/README.md | 37 ++++++++--- docker/rucio_client/build.sh | 10 --- docker/rucio_client/scripts/requirements.txt | 5 ++ 5 files changed, 110 insertions(+), 40 deletions(-) create mode 100644 docker/rucio_client/Dockerfile.minimal delete mode 100755 docker/rucio_client/build.sh create mode 100644 docker/rucio_client/scripts/requirements.txt diff --git a/docker/rucio_client/Dockerfile b/docker/rucio_client/Dockerfile index eedc42f0..3fd30689 100644 --- a/docker/rucio_client/Dockerfile +++ b/docker/rucio_client/Dockerfile @@ -13,10 +13,8 @@ RUN dnf install -y epel-release.noarch && \ RUN dnf config-manager --set-enabled crb # FIXME: What is all this? We can't need it all -RUN dnf install -y python3 python3-pip pylint \ - httpd mod_wsgi libaio gcc python3-devel \ - wget unzip \ - mod_ssl openssl-devel bind-utils nano git && \ +RUN dnf install -y python3 python3-pip libaio gcc python3-devel \ + wget unzip openssl-devel bind-utils nano git && \ dnf clean all && \ rm -rf /var/cache/dnf @@ -35,23 +33,19 @@ RUN dnf -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el RUN dnf config-manager --save --setopt=wlcg.skip_if_unavailable=true \ && dnf install -y which krb5-devel gridsite \ - globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ - gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 \ - nordugrid-arc-client nordugrid-arc-plugins-gfal \ - nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \ - nordugrid-arc-plugins-xrootd \ - xrootd-client && \ + globus-proxy-utils voms-clients-cpp wlcg-voms-cms \ + gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \ dnf clean all && \ rm -rf /var/cache/dnf +COPY docker/rucio_client/scripts /scripts +ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg +ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg # Install Rucio and other stuff - -RUN pip3 install --upgrade pip -RUN pip3 install --upgrade setuptools -RUN pip3 -v install rucio-clients==$RUCIO_VERSION -RUN pip3 install python-gitlab -RUN pip3 install liboidcagent +RUN pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \ + pip3 -v install --no-cache-dir -r /scripts/requirements.txt # Install DAS client to pull info from PhEDEx/DBS RUN curl -k -o /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient @@ -60,12 +54,6 @@ RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root WORKDIR /root RUN git clone https://github.com/rucio/rucio.git -COPY docker/rucio_client/scripts /scripts -COPY docker/rucio_client/loadtest /loadtest - -ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg -ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg - # Pull some useful stuff out of git WORKDIR /root RUN git clone https://github.com/dmwm/CMSRucio.git diff --git a/docker/rucio_client/Dockerfile.minimal b/docker/rucio_client/Dockerfile.minimal new file mode 100644 index 00000000..26d33480 --- /dev/null +++ b/docker/rucio_client/Dockerfile.minimal @@ -0,0 +1,66 @@ +FROM almalinux:9-minimal + +ARG RUCIO_VERSION + +RUN microdnf upgrade -y && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +RUN microdnf install -y epel-release.noarch && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# FIXME: What is all this? We can't need it all +RUN microdnf install -y python3 python3-pip \ + libaio gcc python3-devel \ + wget unzip \ + openssl-devel bind-utils nano git && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +WORKDIR /tmp + +# Get various repos for yum setup +RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/rucio/1.20.3/etc/docker/dev/ca.repo + +# Install WLCG repo for CMS VOMS +RUN rpm -i http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm + +RUN microdnf --setopt=wlcg.skip_if_unavailable=true install -y which krb5-devel gridsite \ + globus-proxy-utils voms-clients-cpp wlcg-voms-cms \ + gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# Install DAS client to pull info from PhEDEx/DBS +RUN curl -k -o /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient +RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt + +WORKDIR /root +RUN git clone https://github.com/rucio/rucio.git + +COPY docker/rucio_client/scripts /scripts + +ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg +ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg + +# Install Rucio and other stuff +RUN pip3 install --upgrade pip setuptools && \ + pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \ + pip3 -v install --no-cache-dir -r /scripts/requirements.txt + +# Pull some useful stuff out of git +WORKDIR /root +RUN git clone https://github.com/dmwm/CMSRucio.git + +RUN mkdir -p /etc/grid-security ; ln -s /cvmfs/grid.cern.ch/etc/grid-security/certificates /etc/grid-security/ + +# Overwrite what's there for testing +# COPY scripts /root/CMSRucio/docker/CMSRucioClient/scripts + +# Our schema +ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python3.9/site-packages/rucio/common/schema/cms.py +RUN python3 -m compileall /usr/lib/python3.9/site-packages/rucio/common/schema + +ENV X509_USER_PROXY=/tmp/x509up +ENTRYPOINT ["/bin/bash", "-c", "sleep infinity"] diff --git a/docker/rucio_client/README.md b/docker/rucio_client/README.md index 72abc0fb..669a1049 100644 --- a/docker/rucio_client/README.md +++ b/docker/rucio_client/README.md @@ -1,15 +1,34 @@ +# CMS rucio_client + N.B. We are in the process of changing over from docker and dockerhub to podmand and CERN's Harbor registry. The documention below reflects this state of flux. For new builds, please switch the build and instructions to Harbor. -Build and run like so where /tmp/x509up is a proxy generated with the DN that matches account [username] - - podman build -t registry.cern.ch/cmsrucio/rucio_client:latest . - podman push registry.cern.ch/cmsrucio/rucio_client:latest - - docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace - docker push ericvaandering/rucio-trace - +## Building +To build the image, the following environment variables need to be set: +``` +export RUCIO_VERSION=32.3.1 +export CMS_VERSION=${RUCIO_VERSION}.cms1 +export HARBOR=registry.cern.ch/cmsrucio +``` + +Then while in the root `CMSRucio` directory, run: +``` +podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile -t $HARBOR/rucio_client:release-$CMS_VERSION . +``` + +To use the `almalinux:9-minimal` base: +``` +podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile.minimal -t $HARBOR/rucio_client:release-minimal-$CMS_VERSION . +``` + +### Old rucio-trace instructions +``` +docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace +docker push ericvaandering/rucio-trace +``` + +## Running To run (no need to build, Eric does this occassionally): docker pull cmssw/rucio_client @@ -19,6 +38,8 @@ To run (no need to build, Eric does this occassionally): docker cp ~/.globus/userkey.pem client:/tmp/userkey.pem docker exec -it client /bin/bash +Grid certs will also need to be added if running locally. + Inside container generate a proxy and connect to the Rucio server chown root *.pem diff --git a/docker/rucio_client/build.sh b/docker/rucio_client/build.sh deleted file mode 100755 index 0f320b26..00000000 --- a/docker/rucio_client/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -export RUCIO_VERSION=32.3.1 -export CMS_VERSION=${RUCIO_VERSION}.cms1 - -export HARBOR=registry.cern.ch/cmsrucio - -podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f Dockerfile -t $HARBOR/rucio-client:release-$CMS_VERSION . \ No newline at end of file diff --git a/docker/rucio_client/scripts/requirements.txt b/docker/rucio_client/scripts/requirements.txt new file mode 100644 index 00000000..2e1b6b40 --- /dev/null +++ b/docker/rucio_client/scripts/requirements.txt @@ -0,0 +1,5 @@ +# for setOneRucioFromGitlab, setRucioFromGitlab, cmslinks.py +python-gitlab~=4.4.0 + +# for syncaccount_oidc.py +liboidcagent~=0.6.0 \ No newline at end of file