Merge pull request #764 from d-ylee/718-enhancement-clean-up-the-clie…

…nt-image

Clean up rucio_client Image and experiment with alma9-minimal client

docker/rucio_client/Dockerfile

@@ -13,10 +13,8 @@ RUN dnf install -y epel-release.noarch && \
 RUN dnf config-manager --set-enabled crb
 
 # FIXME: What is all this? We can't need it all
-RUN dnf install -y python3 python3-pip pylint \
-                   httpd mod_wsgi libaio gcc python3-devel \
-                   wget unzip \
-                   mod_ssl openssl-devel bind-utils nano git && \
+RUN dnf install -y python3 python3-pip libaio gcc python3-devel \
+                   wget unzip openssl-devel bind-utils nano git && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
@@ -35,23 +33,19 @@ RUN dnf -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el
 
 RUN dnf config-manager --save --setopt=wlcg.skip_if_unavailable=true \
         && dnf install -y which krb5-devel gridsite \
-                   globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \
-                   gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 \
-                   nordugrid-arc-client nordugrid-arc-plugins-gfal \
-                   nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \
-                   nordugrid-arc-plugins-xrootd \
-                   xrootd-client && \
+                   globus-proxy-utils voms-clients-cpp wlcg-voms-cms \
+                   gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
+COPY docker/rucio_client/scripts /scripts
+ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg
+ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg
 
 # Install Rucio and other stuff
-
-RUN pip3 install --upgrade pip
-RUN pip3 install --upgrade setuptools
-RUN pip3 -v install rucio-clients==$RUCIO_VERSION
-RUN pip3 install python-gitlab
-RUN pip3 install liboidcagent
+RUN pip3 install --no-cache-dir --upgrade pip setuptools && \
+    pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \
+    pip3 -v install --no-cache-dir -r /scripts/requirements.txt
 
 # Install DAS client to pull info from PhEDEx/DBS
 RUN curl -k -o  /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient
@@ -60,12 +54,6 @@ RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root
 WORKDIR /root
 RUN git clone https://github.com/rucio/rucio.git
 
-COPY docker/rucio_client/scripts /scripts
-COPY docker/rucio_client/loadtest /loadtest
-
-ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg
-ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg
-
 # Pull some useful stuff out of git
 WORKDIR /root
 RUN git clone https://github.com/dmwm/CMSRucio.git

docker/rucio_client/Dockerfile.minimal

@@ -0,0 +1,66 @@
+FROM almalinux:9-minimal
+
+ARG RUCIO_VERSION
+
+RUN microdnf upgrade -y && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+
+RUN microdnf install -y epel-release.noarch && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+
+# FIXME: What is all this? We can't need it all
+RUN microdnf install -y python3 python3-pip \
+                   libaio gcc python3-devel \
+                   wget unzip \
+                   openssl-devel bind-utils nano git && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+
+WORKDIR /tmp
+
+# Get various repos for yum setup
+RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/rucio/1.20.3/etc/docker/dev/ca.repo
+
+# Install WLCG repo for CMS VOMS
+RUN rpm -i http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm 
+
+RUN microdnf --setopt=wlcg.skip_if_unavailable=true install -y which krb5-devel gridsite \
+                   globus-proxy-utils voms-clients-cpp wlcg-voms-cms \
+                   gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \
+    microdnf clean all && \
+    rm -rf /var/cache/dnf
+
+# Install DAS client to pull info from PhEDEx/DBS
+RUN curl -k -o  /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient
+RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt
+
+WORKDIR /root
+RUN git clone https://github.com/rucio/rucio.git
+
+COPY docker/rucio_client/scripts /scripts
+
+ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg
+ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg
+
+# Install Rucio and other stuff
+RUN pip3 install --upgrade pip setuptools && \
+    pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \
+    pip3 -v install --no-cache-dir -r /scripts/requirements.txt
+
+# Pull some useful stuff out of git
+WORKDIR /root
+RUN git clone https://github.com/dmwm/CMSRucio.git
+
+RUN mkdir -p /etc/grid-security ; ln -s /cvmfs/grid.cern.ch/etc/grid-security/certificates /etc/grid-security/
+
+# Overwrite what's there for testing
+# COPY scripts /root/CMSRucio/docker/CMSRucioClient/scripts
+
+# Our schema
+ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python3.9/site-packages/rucio/common/schema/cms.py
+RUN python3 -m compileall /usr/lib/python3.9/site-packages/rucio/common/schema
+
+ENV X509_USER_PROXY=/tmp/x509up
+ENTRYPOINT ["/bin/bash", "-c", "sleep infinity"]

docker/rucio_client/README.md

@@ -1,15 +1,34 @@
+# CMS rucio_client
+
 N.B. We are in the process of changing over from docker and dockerhub to podmand and CERN's Harbor registry. 
 The documention below reflects this state of flux. For new builds, please switch the build and instructions to Harbor.
 
-Build and run like so where /tmp/x509up is a proxy generated with the DN that matches account [username]
-
-    podman build -t registry.cern.ch/cmsrucio/rucio_client:latest .
-    podman push registry.cern.ch/cmsrucio/rucio_client:latest
-
-    docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace
-    docker push ericvaandering/rucio-trace
-
+## Building
+To build the image, the following environment variables need to be set:
+```
+export RUCIO_VERSION=32.3.1
+export CMS_VERSION=${RUCIO_VERSION}.cms1
+export HARBOR=registry.cern.ch/cmsrucio
+```
+
+Then while in the root `CMSRucio` directory, run:
+```
+podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile -t $HARBOR/rucio_client:release-$CMS_VERSION .
+```
+
+To use the `almalinux:9-minimal` base:
+```
+podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile.minimal -t $HARBOR/rucio_client:release-minimal-$CMS_VERSION .
+```
+
+### Old rucio-trace instructions
+```
+docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace
+docker push ericvaandering/rucio-trace
+```
 
+
+## Running 
 To run (no need to build, Eric does this occassionally):
 
     docker pull cmssw/rucio_client
@@ -19,6 +38,8 @@ To run (no need to build, Eric does this occassionally):
     docker cp ~/.globus/userkey.pem client:/tmp/userkey.pem
     docker exec -it client /bin/bash
 
+Grid certs will also need to be added if running locally.
+
 Inside container generate a proxy and connect to the Rucio server
 
     chown root *.pem

docker/rucio_client/scripts/requirements.txt

@@ -0,0 +1,5 @@
+# for setOneRucioFromGitlab, setRucioFromGitlab, cmslinks.py
+python-gitlab~=4.4.0
+
+# for syncaccount_oidc.py
+liboidcagent~=0.6.0