stage-test

!/bin/bash

#usage: stage-test -s system -n name

datetime () { date +"%Y-%m-%d %H:%M:%S"; }

[[ -t 1 ]] && interm=yes

red ()    { echo "\e[1;31m$1\e[m"; }
yellow () { echo "\e[1;33m$1\e[m"; }
log () { printf "$(yellow "[log $(datetime)]") $1\n"; }
err () { printf "$(red "[error $(datetime)]") $1\n"; exit 1; }
exit_without_error () { printf "$(yellow "[log $(datetime)]") $1\n"; exit 0; }

usage () {
    echo "usage: reframe-run -n name -s system -m mount"
    echo ""
    echo "where:"
    echo "  name:        the name of the stack"
    echo "  uarch:       microarchitecture \(one of a100, zen2, zen3, mi200, gh200\)"
    echo "  system:      the cluster name \(e.g. of balfrin, todi, eiger\)"
    echo "  mount:       the mount point of the image"
    echo ""
    [[ "" == "$1" ]] && exit 0
    err "$1"
}

system="-"
name="-"
uarch="-"
mount="-"

ci_path="$CI_PROJECT_DIR"
test_path="$ci_path/$CI_JOB_ID"

while getopts n:s:u:m: flag
do
    case "${flag}" in
        n) name=${OPTARG};;
        s) system=${OPTARG};;
        u) uarch=${OPTARG};;
        m) mount=${OPTARG};;
    esac
done

[[ "-" == "${system}"      ]] && usage "missing system argument"
[[ "-" == "${name}"        ]] && usage "missing name argument"
[[ "-" == "${uarch}"       ]] && usage "missing uarch argument"
[[ "-" == "${mount}"       ]] && usage "missing mount argument"

build_id=${CI_PIPELINE_ID}
pipeline_path=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)

log "SCRATCH        ${SCRATCH}"
log "USER           ${USER}"
log "name           ${name}"
log "system         ${system}"
log "uarch          ${uarch}"
log "mount          ${mount}"
log "build_id       ${build_id}"
log "ci_path        ${ci_path}"
log "test_path      ${test_path}"
log "pipeline_path  ${pipeline_path}"

# configure oras
source ${pipeline_path}/util/setup-oras

# create the path in which tests will run...
mkdir -p ${test_path}
cd ${test_path}
artifact_path="${test_path}/artifacts"
mkdir -p $artifact_path

rego=jfrog.svc.cscs.ch/uenv
#repo_base=build/${system}/${uarch}/${name}/${build_id}
#repo=${repo_base}:latest
repo_base=build/${system}/${uarch}/${name}
repo=${repo_base}:${build_id}

#
# Download the meta data from jfrog
#

# step 1: look up the digest of the meta data
log "looking for meta data for image at $rego/$repo"
meta_digest=$(oras discover --registry-config ${jfrog_creds_path} --output json --artifact-type 'uenv/meta' $rego/$repo | jq -r '.manifests[0].digest')
log "found metadata digest $meta_digest"

# step 2: download the meta data
log "downloading meta data image from ${rego}/${repo_base}@$meta_digest"
oras pull --registry-config ${jfrog_creds_path} --output "${artifact_path}" "${rego}/${repo_base}@$meta_digest"
[[ $? -eq 0  ]] || err "failed to download meta data"

rfm_meta_path="${artifact_path}/meta/extra/reframe.yaml"

log "Using artifacts in $artifact_path"
find "${artifact_path}"

log "looking for reframe metadata in ${rfm_meta_path}"
[[ -f "${rfm_meta_path}" ]] || exit_without_error "no reframe meta data provided: skipping tests"

# copy the meta data to the location where reframe expects to find it.
cp $rfm_meta_path "${artifact_path}/store.yaml"

#
# download the squashfs image from jfrog
#

log "downloading squashfs image from ${rego}/${repo}"
oras pull --registry-config ${jfrog_creds_path} --output "${artifact_path}" "${rego}/${repo}"
[[ $? -eq 0  ]] || err "failed to download squashfs"
squashfs_path="${artifact_path}/store.squashfs"

log "set up a python venv and pip install reframe"
[[ -z "${SLURM_PARTITION}" ]] && RFM_SYSTEM="${system}" || RFM_SYSTEM="${system}:${SLURM_PARTITION}"

rm -rf rfm_venv
python3 -m venv rfm_venv
source rfm_venv/bin/activate
#pip install --upgrade reframe-hpc
git clone https://github.com/reframe-hpc/reframe.git
(cd reframe; git checkout v4.6.1; ./bootstrap.sh)
export PATH="$(pwd)/reframe/bin:$PATH"

log "clone cscs-reframe-tests"
rm -rf cscs-reframe-tests
git clone -b alps https://github.com/eth-cscs/cscs-reframe-tests.git
pip install python-hostlist

export UENV="${squashfs_path}:${mount}"
export RFM_AUTODETECT_METHODS="cat /etc/xthostname,hostname"
export RFM_USE_LOGIN_SHELL=1

REFRAME_COMMAND="reframe -C cscs-reframe-tests/config/cscs.py --report-junit=report.xml -c cscs-reframe-tests/checks/ -r --system=${RFM_SYSTEM}"

log "running reframe: ${REFRAME_COMMAND}"
${REFRAME_COMMAND}
exitcode=$?

if [ "$exitcode" -eq 0 ];
then
    report_path=$(find -type f -name report.xml)
    log "report generated in $report_path"
    cat $report_path
    oras attach --registry-config ${jfrog_creds_path} --artifact-type uenv/reframe "${rego}/${repo}" "${report_path}"
else
    err "Reframe test run failed - no report artifact pushed"
fi

deactivate

exit ${exitcode}