Skip to content

Commit

Permalink
Merge pull request #390 from boegel/2023.06-software.eessi.io_eessi-c…
Browse files Browse the repository at this point in the history 
…ontainer-script
  • Loading branch information
@bedroge
bedroge authored Nov 25, 2023
2 parents 86fa918 + d04f9bb commit 006aec4
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 38 deletions.
16 changes: 8 additions & 8 deletions .github/workflows/test_eessi_container_script.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
outfile=out_listrepos.txt
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI-pilot" ${outfile}
grep "EESSI" ${outfile}
# test use of --list-repos with custom repos.cfg
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
Expand All @@ -57,7 +57,7 @@ jobs:
echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
echo "repo_version = 20HT.TP" >> cfg/repos.cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI-pilot" ${outfile}
grep "EESSI" ${outfile}
export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile2}
Expand Down Expand Up @@ -90,15 +90,15 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
outfile=out_readwrite.txt
fn="test_${RANDOM}.txt"
echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
chmod u+x test_script.sh
export SINGULARITY_BIND="$PWD:/test"
./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
# note: must use '--access rw' again here, since touched file is in overlay upper dir
./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
grep "/cvmfs/software.eessi.io/${fn}$" $outfile
# test use of --resume
elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
Expand All @@ -120,12 +120,12 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
outfile=out_save.txt
fn="test_${RANDOM}.txt"
test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
test_cmd="touch /cvmfs/software.eessi.io/${fn}"
./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
rm -f ${outfile}
./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
grep "/cvmfs/software.eessi.io/${fn}$" $outfile
tar tfv test-save.tar | grep "overlay-upper/${fn}"
Expand Down
24 changes: 12 additions & 12 deletions create_tarball.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ set -e
base_dir=$(dirname $(realpath $0))

if [ $# -ne 4 ]; then
echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <pilot version (example: 2021.03)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <version (example: 2023.06)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
exit 1
fi
eessi_tmpdir=$1
pilot_version=$2
eessi_version=$2
cpu_arch_subdir=$3
target_tgz=$4

Expand All @@ -20,15 +20,15 @@ os="linux"
source ${base_dir}/init/eessi_defaults
cvmfs_repo=${EESSI_CVMFS_REPO}

software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
if [ ! -d ${software_dir} ]; then
echo "Software directory ${software_dir} does not exist?!" >&2
exit 2
fi

overlay_upper_dir="${eessi_tmpdir}/overlay-upper"

software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
if [ ! -d ${software_dir_overlay} ]; then
echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2
exit 3
Expand All @@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
files_list=${tmpdir}/files.list.txt
module_files_list=${tmpdir}/module_files.list.txt

if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
# include Lmod cache and configuration file (lmodrc.lua),
# skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
fi
if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
# module files
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
# module symlinks
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
# module files and symlinks
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
| grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
>> ${module_files_list}
fi
if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
# installation directories but only those for which module files were created
# Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
# using EasyBuild's standard module naming scheme) match the name of the
Expand All @@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu
# installation directories), the procedure will likely not work.
for package_version in $(cat ${module_files_list}); do
echo "handling ${package_version}"
ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
| grep -v '/\.wh\.' >> ${files_list}
done
fi
Expand Down
40 changes: 22 additions & 18 deletions eessi_container.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ display_help() {
echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or"
echo " MODE==run (run a script or command) [default: shell]"
echo " -r | --repository CFG - configuration file or identifier defining the"
echo " repository to use [default: EESSI-pilot via"
echo " repository to use [default: EESSI via"
echo " default container, see --container]"
echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
echo " where DIR points to a previously used tmp directory"
Expand Down Expand Up @@ -111,7 +111,7 @@ VERBOSE=0
STORAGE=
LIST_REPOS=0
MODE="shell"
REPOSITORY="EESSI-pilot"
REPOSITORY="EESSI"
RESUME=
SAVE=
HTTP_PROXY=${http_proxy:-}
Expand Down Expand Up @@ -194,7 +194,7 @@ set -- "${POSITIONAL_ARGS[@]}"

if [[ ${LIST_REPOS} -eq 1 ]]; then
echo "Listing available repositories with format 'name [source]':"
echo " EESSI-pilot [default]"
echo " EESSI [default]"
if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then
cfg_load ${EESSI_REPOS_CFG_FILE}
sections=$(cfg_sections)
Expand Down Expand Up @@ -226,7 +226,7 @@ fi

# TODO (arg -r|--repository) check if repository is known
# REPOSITORY_ERROR_EXITCODE
if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
fi

Expand Down Expand Up @@ -403,7 +403,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
# set up repository config (always create directory repos_cfg and populate it with info when
# arg -r|--repository is used)
mkdir -p ${EESSI_TMPDIR}/repos_cfg
if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
if [[ "${REPOSITORY}" == "EESSI" ]]; then
# need to source defaults as late as possible (see other sourcing below)
source ${TOPDIR}/init/eessi_defaults

Expand All @@ -427,7 +427,7 @@ else
# map { local_filepath -> container_filepath }
#
# repo_name_domain is the domain part of the repo_name, e.g.,
# eessi-hpc.org for pilot.eessi-hpc.org
# eessi.io for software.eessi.io
#
# where config bundle includes the files (-> target location in container)
# - default.local -> /etc/cvmfs/default.local
Expand Down Expand Up @@ -479,7 +479,7 @@ else
target=${cfg_file_map[${src}]}
BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
done
export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
export EESSI_VERSION_OVERRIDE=${repo_version}
export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
# need to source defaults as late as possible (after *_OVERRIDEs)
source ${TOPDIR}/init/eessi_defaults
Expand Down Expand Up @@ -513,10 +513,14 @@ fi
# 4. set up vars and dirs specific to a scenario

declare -a EESSI_FUSE_MOUNTS=()

# always mount cvmfs-config repo (to get access to software.eessi.io)
EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch")

if [[ "${ACCESS}" == "ro" ]]; then
export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"
export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
export EESSI_FUSE_MOUNTS
fi

Expand All @@ -525,18 +529,18 @@ if [[ "${ACCESS}" == "rw" ]]; then
mkdir -p ${EESSI_TMPDIR}/overlay-work

# set environment variables for fuse mounts in Singularity container
export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")

EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
export EESSI_PILOT_WRITABLE_OVERLAY
EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
export EESSI_WRITABLE_OVERLAY

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}")
export EESSI_FUSE_MOUNTS
fi

Expand Down

0 comments on commit 006aec4

Please sign in to comment.