Add alma9 images (old style, we need ythem for upcoming training next week)

.github/workflows/dockercd-dev.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image: [cc-base-ubuntu, cc-analysis-ubuntu, cc-base-centos7, cc-analysis-centos7]
+        image: [cc-base-ubuntu, cc-analysis-ubuntu, cc-dask-alma8, cc-analysis-alma8, cc-dask-alma9, cc-analysis-alma9]
         # Keep this line in sync with gh actions @ coffea-dask repo
         # python: [3.8, 3.9, '3.10']
         #exclude:

.github/workflows/dockerci-dev.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image: [cc-base-ubuntu, cc-analysis-ubuntu, cc-base-alma8, cc-analysis-alma8]
+        image: [cc-base-ubuntu, cc-analysis-ubuntu, cc-dask-alma8, cc-analysis-alma8, cc-dask-alma9, cc-analysis-alma9]
         # Keep this line in sync with gh actions @ coffea-dask repo
         # python: [3.8, 3.9, '3.10']
         #exclude:

docker/Dockerfile.cc-analysis-alma9

@@ -0,0 +1,220 @@
+#FROM coffeateam/coffea-base-almalinux8:0.7.22-py3.10
+FROM coffeateam/coffea-dask-almalinux9:latest-py3.10
+
+USER root
+LABEL maintainer="Oksana Shadura <[email protected]>"
+# Jupyterhub
+ARG TAG="development"
+ARG NB_USER="cms-jovyan"
+ARG NB_UID="6440"
+ARG NB_GID="11265"
+ARG CERT_DIR="/etc/cmsaf-secrets"
+# Hack for GH Actions
+ARG GITHUB_ACTIONS="false"
+# FIX ME AFTER TEST:
+#ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG BEARER_TOKEN_FILE="/tmp/.xcache/access_token"
+ARG SEC_TOKEN_SYSTEM_DIRECTORY="/tmp/.condor"
+ARG XCACHE_HOST="xcache"
+#ARG XCACHE_HOST="red-xcache1.unl.edu"
+
+
+# Configure environment
+ENV CONDA_DIR /usr/local
+ENV CERT_DIR $CERT_DIR
+ENV XCACHE_HOST $XCACHE_HOST
+ENV BEARER_TOKEN_FILE $BEARER_TOKEN_FILE
+ENV SEC_TOKEN_SYSTEM_DIRECTORY $SEC_TOKEN_SYSTEM_DIRECTORY
+ENV SHELL /bin/bash
+ENV NB_USER $NB_USER
+ENV USER $NB_USER
+ENV NB_UID $NB_UID
+ENV NB_GID $NB_GID
+ENV HOME /home/$NB_USER
+ENV PATH "${CONDA_DIR}/bin/:$PATH"
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+RUN yum -y update \
+ && yum -y group install "Development Tools" \
+ && yum -y install \
+    wget \
+    epel-release \
+    gettext \
+    bzip2 \
+    ca-certificates \
+    sudo \
+    langpacks-en \
+    glibc-langpack-en \
+    glibc-all-langpacks \
+    liberation-fonts \
+    wget \
+    nss_wrapper \
+    openssl \
+    emacs \
+    git \
+    unzip \
+    nano \
+    vim  \
+    net-tools \
+    libXext \
+    libSM \
+    libXrender \
+    nc \
+    openssh-clients \
+    tzdata \
+    unzip \
+    gdb \
+    && yum clean all && rm -rf /var/cache/yum
+
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-amd64 /usr/bin/tini
+RUN chmod +x /usr/bin/tini
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY jupyterhub/fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+RUN rm -rf /etc/grid-security && \
+    ln -s /usr/local/etc/grid-security /etc/grid-security && \
+    chmod 755 /etc/grid-security/certificates && \
+    chmod g-w /etc/grid-security/certificates
+
+# Enable prompt color in the skeleton .bashrc before creating the default NB_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create NB_USER with name jovyan user with given UID and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    groupadd -r "${NB_USER}" --gid "$NB_GID" && \
+    useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    mkdir -p "${CERT_DIR}" && \
+    mkdir -p /var/lib/condor && \
+    chown -R "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \
+    chown -R "${NB_USER}:${NB_GID}" "${HOME}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER $NB_USER
+WORKDIR $HOME
+# Setup work directory for backward-compatibility
+RUN mkdir $HOME/work && \
+    mkdir -p $HOME/.local && \
+    mkdir -p $HOME/.condor/tokens.d && \
+    fix-permissions $HOME
+
+RUN mamba remove htcondor
+
+#Dask dependencies and HTCondor
+RUN mamba install --yes \
+    -c conda-forge \
+    htcondor==10.8.0 \
+    xgboost \
+    pyhf \
+    xrootd \
+    scipy>=1.8.1 \
+    cabinetry \
+    vector \
+    hist \
+    mplhep \
+    iminuit \
+    cmake \
+    ndcctools \
+    && mamba clean \
+        --all \
+        --force-pkgs-dirs \
+        --yes
+
+RUN pip install --no-cache-dir \
+    aiostream \
+    supervisor \
+    correctionlib \
+    funcx \
+    pyyaml \
+    # ML packages
+    dask-ml \
+    dask-gateway \
+    prometheus_client \
+    comm>=0.1.2 \
+    mlflow
+
+#RUN if [ "${TAG:-}" == "development" ]; then pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa ; else pip install --no-cache-dir coffea_casa -U ; fi
+RUN pip install --no-cache-dir git+https://github.com/CoffeaTeam/coffea-casa.git#egg=coffea_casa
+
+# ------- xrootd-xcache-plugin -------------------------------
+RUN cd /tmp && \
+    git clone -b xcache https://github.com/jthiltges/xrdcl-authz-plugin.git && \
+    cd xrdcl-authz-plugin && \
+    mkdir build && \
+    cd  build && \
+    cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
+    make && \
+    make install
+
+ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
+ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClXcachePlugin-5.so"
+
+# Include additional CA certificates beyond ca-policy-lcg
+COPY certs/* /etc/grid-security/certificates/
+RUN openssl rehash /etc/grid-security/certificates/
+
+# TODO: RETEST IF WE STILL NEED THIS 
+ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
+ENV PATH="${CONDA_DIR}/bin/:$PATH"
+
+USER root
+# Setup supervisord files
+COPY k8s-worker/supervisord.conf  /etc/supervisor/
+# Setup HTCondor user/group and change group for user $NB_USER
+# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and
+# it configured from kubernetes side and updated in docker container to match it
+RUN groupadd -r condor && \
+    useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor
+
+# FIXME: merge PRs open in distributed.git (oshadura)
+# Distributed: we need to install patched version of distributed version
+COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed
+RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \
+    patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \
+    patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch
+    # && patch -p2 < 0003-Activate-patch.patch
+    # && patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch
+    # && patch -p2 < 0005-Add-nanny-patch.patch
+
+# FIXME: we have a wrong path, let's make a link.
+# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
+# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
+RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin
+
+# Cleanup
+RUN rm -rf /tmp/* \
+    && rm -rf $HOME/.cache/.pip/* \
+    && mamba clean --all -f -y \
+    && find ${CONDA_DIR} -type f -name '*.a' -delete \
+    && find ${CONDA_DIR} -type f -name '*.pyc' -delete \
+    && find ${CONDA_DIR} -type f -name '*.js.map' -delete \
+    && (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
+    && rm -rf ${CONDA_DIR}/pkgs
+
+# Mkdir CVMFS mount directory.  Mount done upstream in configs. Harmless
+# if not utilized, crucial if CVMFS is desired
+RUN mkdir /cvmfs
+
+# Prepare HTCondor spec. environment and execute dask-worker command
+# If we using this container as a sidecar, we don't setup any HTCondor spec. environment
+# nor and execute dask-worker command
+ADD prepare-env/prepare-env-cc-analysis.sh /usr/local/bin/prepare-env.sh
+RUN chmod ugo+x /usr/local/bin/prepare-env.sh
+
+USER $NB_USER
+ENTRYPOINT ["tini", "-g", "--", "/usr/local/bin/prepare-env.sh"]