diff --git a/docker/rucio_client/Dockerfile b/docker/rucio_client/Dockerfile index eedc42f..3fd3068 100644 --- a/docker/rucio_client/Dockerfile +++ b/docker/rucio_client/Dockerfile @@ -13,10 +13,8 @@ RUN dnf install -y epel-release.noarch && \ RUN dnf config-manager --set-enabled crb # FIXME: What is all this? We can't need it all -RUN dnf install -y python3 python3-pip pylint \ - httpd mod_wsgi libaio gcc python3-devel \ - wget unzip \ - mod_ssl openssl-devel bind-utils nano git && \ +RUN dnf install -y python3 python3-pip libaio gcc python3-devel \ + wget unzip openssl-devel bind-utils nano git && \ dnf clean all && \ rm -rf /var/cache/dnf @@ -35,23 +33,19 @@ RUN dnf -y install http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el RUN dnf config-manager --save --setopt=wlcg.skip_if_unavailable=true \ && dnf install -y which krb5-devel gridsite \ - globus-proxy-utils voms-clients-java voms-clients-cpp wlcg-voms-cms \ - gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 \ - nordugrid-arc-client nordugrid-arc-plugins-gfal \ - nordugrid-arc-plugins-globus nordugrid-arc-plugins-s3 \ - nordugrid-arc-plugins-xrootd \ - xrootd-client && \ + globus-proxy-utils voms-clients-cpp wlcg-voms-cms \ + gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \ dnf clean all && \ rm -rf /var/cache/dnf +COPY docker/rucio_client/scripts /scripts +ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg +ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg # Install Rucio and other stuff - -RUN pip3 install --upgrade pip -RUN pip3 install --upgrade setuptools -RUN pip3 -v install rucio-clients==$RUCIO_VERSION -RUN pip3 install python-gitlab -RUN pip3 install liboidcagent +RUN pip3 install --no-cache-dir --upgrade pip setuptools && \ + pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \ + pip3 -v install --no-cache-dir -r /scripts/requirements.txt # Install DAS client to pull info from PhEDEx/DBS RUN curl -k -o /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient @@ -60,12 +54,6 @@ RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root WORKDIR /root RUN git clone https://github.com/rucio/rucio.git -COPY docker/rucio_client/scripts /scripts -COPY docker/rucio_client/loadtest /loadtest - -ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg -ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg - # Pull some useful stuff out of git WORKDIR /root RUN git clone https://github.com/dmwm/CMSRucio.git diff --git a/docker/rucio_client/Dockerfile.minimal b/docker/rucio_client/Dockerfile.minimal new file mode 100644 index 0000000..26d3348 --- /dev/null +++ b/docker/rucio_client/Dockerfile.minimal @@ -0,0 +1,66 @@ +FROM almalinux:9-minimal + +ARG RUCIO_VERSION + +RUN microdnf upgrade -y && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +RUN microdnf install -y epel-release.noarch && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# FIXME: What is all this? We can't need it all +RUN microdnf install -y python3 python3-pip \ + libaio gcc python3-devel \ + wget unzip \ + openssl-devel bind-utils nano git && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +WORKDIR /tmp + +# Get various repos for yum setup +RUN curl -o /etc/yum.repos.d/ca.repo https://raw.githubusercontent.com/rucio/rucio/1.20.3/etc/docker/dev/ca.repo + +# Install WLCG repo for CMS VOMS +RUN rpm -i http://linuxsoft.cern.ch/wlcg/el9/x86_64/wlcg-repo-1.0.0-1.el9.noarch.rpm + +RUN microdnf --setopt=wlcg.skip_if_unavailable=true install -y which krb5-devel gridsite \ + globus-proxy-utils voms-clients-cpp wlcg-voms-cms \ + gfal2-util gfal2-all gfal2-plugin-xrootd python3-gfal2 && \ + microdnf clean all && \ + rm -rf /var/cache/dnf + +# Install DAS client to pull info from PhEDEx/DBS +RUN curl -k -o /bin/dasgoclient https://www.lns.cornell.edu/~vk/Work/dasgoclient && chmod +x /bin/dasgoclient +RUN curl -o /etc/ca.crt https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt + +WORKDIR /root +RUN git clone https://github.com/rucio/rucio.git + +COPY docker/rucio_client/scripts /scripts + +ADD docker/rucio_client/rucio-int.cfg /opt/rucio-int/etc/rucio.cfg +ADD docker/rucio_client/rucio-prod.cfg /opt/rucio-prod/etc/rucio.cfg + +# Install Rucio and other stuff +RUN pip3 install --upgrade pip setuptools && \ + pip3 -v install --no-cache-dir rucio-clients==$RUCIO_VERSION && \ + pip3 -v install --no-cache-dir -r /scripts/requirements.txt + +# Pull some useful stuff out of git +WORKDIR /root +RUN git clone https://github.com/dmwm/CMSRucio.git + +RUN mkdir -p /etc/grid-security ; ln -s /cvmfs/grid.cern.ch/etc/grid-security/certificates /etc/grid-security/ + +# Overwrite what's there for testing +# COPY scripts /root/CMSRucio/docker/CMSRucioClient/scripts + +# Our schema +ADD https://raw.githubusercontent.com/ericvaandering/rucio/cms_nano12/lib/rucio/common/schema/cms.py /usr/lib/python3.9/site-packages/rucio/common/schema/cms.py +RUN python3 -m compileall /usr/lib/python3.9/site-packages/rucio/common/schema + +ENV X509_USER_PROXY=/tmp/x509up +ENTRYPOINT ["/bin/bash", "-c", "sleep infinity"] diff --git a/docker/rucio_client/README.md b/docker/rucio_client/README.md index 72abc0f..669a104 100644 --- a/docker/rucio_client/README.md +++ b/docker/rucio_client/README.md @@ -1,15 +1,34 @@ +# CMS rucio_client + N.B. We are in the process of changing over from docker and dockerhub to podmand and CERN's Harbor registry. The documention below reflects this state of flux. For new builds, please switch the build and instructions to Harbor. -Build and run like so where /tmp/x509up is a proxy generated with the DN that matches account [username] - - podman build -t registry.cern.ch/cmsrucio/rucio_client:latest . - podman push registry.cern.ch/cmsrucio/rucio_client:latest - - docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace - docker push ericvaandering/rucio-trace - +## Building +To build the image, the following environment variables need to be set: +``` +export RUCIO_VERSION=32.3.1 +export CMS_VERSION=${RUCIO_VERSION}.cms1 +export HARBOR=registry.cern.ch/cmsrucio +``` + +Then while in the root `CMSRucio` directory, run: +``` +podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile -t $HARBOR/rucio_client:release-$CMS_VERSION . +``` + +To use the `almalinux:9-minimal` base: +``` +podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f docker/rucio_client/Dockerfile.minimal -t $HARBOR/rucio_client:release-minimal-$CMS_VERSION . +``` + +### Old rucio-trace instructions +``` +docker build . -f Dockerfile.trace -t ericvaandering/rucio-trace +docker push ericvaandering/rucio-trace +``` + +## Running To run (no need to build, Eric does this occassionally): docker pull cmssw/rucio_client @@ -19,6 +38,8 @@ To run (no need to build, Eric does this occassionally): docker cp ~/.globus/userkey.pem client:/tmp/userkey.pem docker exec -it client /bin/bash +Grid certs will also need to be added if running locally. + Inside container generate a proxy and connect to the Rucio server chown root *.pem diff --git a/docker/rucio_client/build.sh b/docker/rucio_client/build.sh deleted file mode 100755 index 0f320b2..0000000 --- a/docker/rucio_client/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -export RUCIO_VERSION=32.3.1 -export CMS_VERSION=${RUCIO_VERSION}.cms1 - -export HARBOR=registry.cern.ch/cmsrucio - -podman build --build-arg RUCIO_VERSION=$RUCIO_VERSION -f Dockerfile -t $HARBOR/rucio-client:release-$CMS_VERSION . \ No newline at end of file diff --git a/docker/rucio_client/scripts/requirements.txt b/docker/rucio_client/scripts/requirements.txt new file mode 100644 index 0000000..2e1b6b4 --- /dev/null +++ b/docker/rucio_client/scripts/requirements.txt @@ -0,0 +1,5 @@ +# for setOneRucioFromGitlab, setRucioFromGitlab, cmslinks.py +python-gitlab~=4.4.0 + +# for syncaccount_oidc.py +liboidcagent~=0.6.0 \ No newline at end of file