Merge pull request #57 from ACCESS-NRI/upstream_merge

Merge Upstream

.github/workflows/build_and_test.yml

@@ -3,10 +3,24 @@ on: pull_request
 jobs:
   build:
     runs-on: ubuntu-latest
+    container:
+      image: quay.io/singularity/singularity:v3.11.4
+      options: --privileged
     steps:
       - name: Checkout repository
         ### Latest at time of writing
         uses: actions/[email protected]
+      - name: Check if container definition has changed
+        id: changed-container-def
+        uses: tj-actions/changed-files@v37
+        with:
+          files_yaml: |
+            containerdef:
+              - container/container.def
+      - name: Build container if definition has changed
+        if: steps.changed-container-def.outputs.containerdef_any_changed == 'true'
+        run: |
+          sudo -E singularity build container/base.sif container/container.def
       - name: Sync repository to Gadi
         ### Latest at time of writing
         uses: up9cloud/[email protected]
@@ -38,7 +52,7 @@ jobs:
             eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
             eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
             cd $JOB_LOG_DIR
-            qsub -N build_conda -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/xp65 -v SCRIPT_DIR -P iq82 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh
+            qsub -N build_conda -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/xp65+gdata/hh5 -v SCRIPT_DIR -P iq82 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh
   test:
     needs: build
     runs-on: ubuntu-latest
@@ -54,4 +68,4 @@ jobs:
             eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
             eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
             cd $JOB_LOG_DIR
-            qsub -N test_conda -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/xp65 -v SCRIPT_DIR -P iq82 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh
+            qsub -N test_conda -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/xp65+gdata/hh5 -v SCRIPT_DIR -P iq82 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh

container/container.def

@@ -1,38 +1,45 @@
-%setup
-        mkdir -p ${SINGULARITY_ROOTFS}/usr/bin
-        ln -s usr/bin ${SINGULARITY_ROOTFS}/bin
-        ln -s half-root/boot ${SINGULARITY_ROOTFS}/boot
-        ln -s half-root/etc ${SINGULARITY_ROOTFS}/etc
-        ln -s half-root/images ${SINGULARITY_ROOTFS}/images
-        ln -s half-root/initrd ${SINGULARITY_ROOTFS}/initrd
-        ln -s usr/lib ${SINGULARITY_ROOTFS}/lib
-        ln -s usr/lib64 ${SINGULARITY_ROOTFS}/lib64
-        ln -s usr/sbin ${SINGULARITY_ROOTFS}/sbin
-
-        mkdir -p ${SINGULARITY_ROOTFS}/opt
-        ln -s ../half-root/opt/Modules ${SINGULARITY_ROOTFS}/opt/Modules
-        ln -s ../half-root/opt/beegfs ${SINGULARITY_ROOTFS}/opt/beegfs
-        ln -s ../half-root/opt/bin ${SINGULARITY_ROOTFS}/opt/bin
-        ln -s ../half-root/opt/gdrcopy ${SINGULARITY_ROOTFS}/opt/gdrcopy
-        ln -s ../half-root/opt/intel ${SINGULARITY_ROOTFS}/opt/intel
-        ln -s ../half-root/opt/knem-1.1.4.90mlnx1 ${SINGULARITY_ROOTFS}/opt/knem-1.1.4.90mlnx1
-        ln -s ../half-root/opt/mellanox ${SINGULARITY_ROOTFS}/opt/mellanox
-        ln -s ../half-root/opt/pbs ${SINGULARITY_ROOTFS}/opt/pbs
-        ln -s ../half-root/opt/rash ${SINGULARITY_ROOTFS}/opt/rash
-        ln -s ../half-root/opt/rh ${SINGULARITY_ROOTFS}/opt/rh
-
-        ### Mount points for various bits of the image
-        mkdir -p ${SINGULARITY_ROOTFS}/etc
-        mkdir -p ${SINGULARITY_ROOTFS}/half-root
-        mkdir -p ${SINGULARITY_ROOTFS}/local
-        mkdir -p ${SINGULARITY_ROOTFS}/ram
-        mkdir -p ${SINGULARITY_ROOTFS}/run
-        mkdir -p ${SINGULARITY_ROOTFS}/system
-        mkdir -p ${SINGULARITY_ROOTFS}/usr
-
-        ### Mount points for service sockets
-        mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss
-        mkdir -p ${SINGULARITY_ROOTFS}/var/run/munge
-
-%runscript
+Bootstrap: scratch
+
+%setup
+        mkdir -p ${SINGULARITY_ROOTFS}/usr/bin
+        ln -s usr/bin ${SINGULARITY_ROOTFS}/bin
+        ln -s half-root/boot ${SINGULARITY_ROOTFS}/boot
+        ln -s half-root/etc ${SINGULARITY_ROOTFS}/etc
+        ln -s half-root/images ${SINGULARITY_ROOTFS}/images
+        ln -s half-root/initrd ${SINGULARITY_ROOTFS}/initrd
+        ln -s usr/lib ${SINGULARITY_ROOTFS}/lib
+        ln -s usr/lib64 ${SINGULARITY_ROOTFS}/lib64
+        ln -s usr/sbin ${SINGULARITY_ROOTFS}/sbin
+
+        mkdir -p ${SINGULARITY_ROOTFS}/opt
+        ln -s ../half-root/opt/Modules ${SINGULARITY_ROOTFS}/opt/Modules
+        ln -s ../half-root/opt/beegfs ${SINGULARITY_ROOTFS}/opt/beegfs
+        ln -s ../half-root/opt/bin ${SINGULARITY_ROOTFS}/opt/bin
+        ln -s ../half-root/opt/gdrcopy ${SINGULARITY_ROOTFS}/opt/gdrcopy
+        ln -s ../half-root/opt/intel ${SINGULARITY_ROOTFS}/opt/intel
+        ln -s ../half-root/opt/knem-1.1.4.90mlnx1 ${SINGULARITY_ROOTFS}/opt/knem-1.1.4.90mlnx1
+        ln -s ../half-root/opt/mellanox ${SINGULARITY_ROOTFS}/opt/mellanox
+        ln -s ../half-root/opt/pbs ${SINGULARITY_ROOTFS}/opt/pbs
+        ln -s ../half-root/opt/rash ${SINGULARITY_ROOTFS}/opt/rash
+        ln -s ../half-root/opt/rh ${SINGULARITY_ROOTFS}/opt/rh
+
+        ### Mount points for various bits of the image
+        mkdir -p ${SINGULARITY_ROOTFS}/etc
+        mkdir -p ${SINGULARITY_ROOTFS}/half-root
+        mkdir -p ${SINGULARITY_ROOTFS}/local
+        mkdir -p ${SINGULARITY_ROOTFS}/ram
+        mkdir -p ${SINGULARITY_ROOTFS}/run
+        mkdir -p ${SINGULARITY_ROOTFS}/system
+        mkdir -p ${SINGULARITY_ROOTFS}/usr
+
+        ### Mount points for service sockets
+        mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss
+        mkdir -p ${SINGULARITY_ROOTFS}/var/run/munge
+
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.07
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.10
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.01
+        mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.04
+
+%runscript
         /usr/bin/bash -l

scripts/build.sh

@@ -60,17 +60,30 @@ function inner() {
 
     ### Replace things from apps
     for pkg in "${replace_from_apps[@]}"; do
-        for dir in bin etc lib include; do 
-            pushd $dir 
-            for i in $( find /apps/$pkg/$dir -maxdepth 1 -type f ); do 
-                fn=$( basename $i ) 
-                [[ -e $fn ]] && rm $fn && ln -s $i
-            done
-            popd
+        for dir in bin etc lib include; do
+	    if [[ -d "${dir}" ]]; then
+                pushd $dir
+                apps_subdir=/apps/"${pkg}"/"${dir}"
+                for i in $( find "${apps_subdir}" -type f ); do
+                    fn="${i//$apps_subdir\//}"
+                    [[ -e $fn ]] && rm $fn
+                    [[ "${fn}" != "${fn%/*}" ]] && mkdir -p "${fn%/*}"
+                    ln -s "${i}" "${fn}"
+                done
+                popd
+	    fi
         done
     done
     popd
 
+    ### Update any supporting infrastructure
+    copy_if_changed "${SCRIPT_DIR}"/launcher.sh "${CONDA_SCRIPT_PATH}"/launcher.sh
+    for override in "${SCRIPT_DIR}"/overrides/*; do
+        copy_if_changed "${override}" "${CONDA_SCRIPT_PATH}"/overrides/"${override##*/}"
+    done
+    copy_and_replace_if_changed "${SCRIPT_DIR}"/../modules/common_v3 "${CONDA_MODULE_PATH}"/.common_v3       CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME SCRIPT_SUBDIR
+    copy_and_replace_if_changed "${SCRIPT_DIR}"/launcher_conf.sh     "${CONDA_SCRIPT_PATH}"/launcher_conf.sh CONDA_BASE APPS_SUBDIR CONDA_INSTALL_BASENAME
+
     ### Create symlink tree
     mkdir -p "${CONDA_SCRIPT_PATH}"/"${FULLENV}".d/{bin,overrides}
     cp "${CONDA_SCRIPT_PATH}"/{launcher.sh,launcher_conf.sh} "${CONDA_SCRIPT_PATH}"/"${FULLENV}".d/bin
@@ -120,6 +133,13 @@ else
     ./initialise.sh
 fi
 
+### Copy in any files outside the conda directory tree that may be needed
+echo "Copying external files"
+for f in "${outside_files_to_copy[@]}"; do
+    mkdir -p "${OVERLAY_BASE}"/$( dirname "${f#/g/}" )
+    cp "${f}" "${OVERLAY_BASE}"/"${f#/g/}"
+done
+
 if [[ -e  "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}.sqsh" ]]; then
     pushd "${CONDA_TEMP_PATH}"
     unsquashfs -processors 1 "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}.sqsh"
@@ -133,15 +153,23 @@ fi
 
 ln -sf "${ENV_INSTALLATION_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/
 
-"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g "${CONTAINER_PATH}" $( realpath $0 ) --inner "${DO_UPDATE}"
+if [[ -e "${CONTAINER_PATH}" ]]; then
+    ### New container, use that
+    my_container="${CONTAINER_PATH}"
+else
+    my_container="${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}"
+fi
+
+"${SINGULARITY_BINARY_PATH}" -s exec --bind /etc,/half-root,/local,/ram,/run,/system,/usr,/var/lib/sss,/var/run/munge,/var/lib/rpm,"${OVERLAY_BASE}":/g "${my_container}" $( realpath $0 ) --inner "${DO_UPDATE}"
 if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
 ### See if the container has been updated
-read newhash fn < <( md5sum "${CONTAINER_PATH}" )
-read oldhash fn < <( md5sum "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}" )
-if [[ "${oldhash}" != "${newhash}" ]]; then
+### The container will only exist on ${CONTAINER_PATH} if it was built by the github action
+#read newhash fn < <( md5sum "${CONTAINER_PATH}" )
+#read oldhash fn < <( md5sum "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}" )
+if [[ -e "${CONTAINER_PATH}" ]]; then
     echo "Container update detected. Copying in new container"
     cp "${CONTAINER_PATH}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/etc/"${CONTAINER_PATH##*/}"
 fi
@@ -162,15 +190,15 @@ pushd "${CONDA_TEMP_PATH}"
 ### Don't need to think too hard, squashfs are read-only
 chgrp -R "${APPS_USERS_GROUP}" squashfs-root
 
-mksquashfs squashfs-root "${FULLENV}".sqsh -b 1M -no-recovery -noI -noD -noF -noX -processors 8 2>/dev/null
+mksquashfs squashfs-root "${FULLENV}".sqsh -no-fragments -no-duplicates -no-sparse -no-exports -no-recovery -noI -noD -noF -noX -processors 8 2>/dev/null
 ### Stage this file and rename when we're ready
 cp "${FULLENV}".sqsh "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp
 set_apps_perms "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp
 popd
 
 ### Can't use ${CONDA_SCRIPT_PATH} or "${CONDA_INSTALLATION_PATH}" due to the need to string match on those paths
 ### which they won't with the '/./' part required for arcane rsync magic
-construct_module_insert "${SINGULARITY_BINARY_PATH}" "${OVERLAY_BASE}" "${CONTAINER_PATH}" "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${SCRIPT_DIR}"/condaenv.sh "${CONDA_INSTALLATION_PATH}" "${CONDA_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}" "${CONDA_BASE}"/"${SCRIPT_SUBDIR}"/"${FULLENV}".d/bin "${CONDA_OUTER_BASE}"/"${MODULE_SUBDIR}"/"${MODULE_NAME}"/."${FULLENV}"
+construct_module_insert "${SINGULARITY_BINARY_PATH}" "${OVERLAY_BASE}" "${my_container}" "${BUILD_STAGE_DIR}"/"${FULLENV}".sqsh.tmp "${SCRIPT_DIR}"/condaenv.sh "${CONDA_INSTALLATION_PATH}" "${CONDA_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}" "${CONDA_BASE}"/"${SCRIPT_SUBDIR}"/"${FULLENV}".d/bin "${CONDA_OUTER_BASE}"/"${MODULE_SUBDIR}"/"${MODULE_NAME}"/."${FULLENV}"
 
 rm "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/"${FULLENV}"
 ln -s /opt/conda/"${FULLENV}" "${CONDA_OUTER_BASE}"/"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}"/envs/