From 20600fe6f62ce3d800a20ff8dfb7309e89a69851 Mon Sep 17 00:00:00 2001 From: Steve Goldhaber Date: Fri, 21 Jul 2023 21:03:57 +0200 Subject: [PATCH] Change aux_cam to aux_cam_noresm and introduce script to run tests --- cime_config/testdefs/testlist_cam.xml | 16 +- test/system/run_noresm_tests.sh | 402 ++++++++++++++++ test/system/test_driver.sh | 665 -------------------------- 3 files changed, 410 insertions(+), 673 deletions(-) create mode 100755 test/system/run_noresm_tests.sh delete mode 100755 test/system/test_driver.sh diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index cde11996ef..d1f05ea1e2 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -12,7 +12,7 @@ - + @@ -21,7 +21,7 @@ - + @@ -50,7 +50,7 @@ - + @@ -59,7 +59,7 @@ - + @@ -73,7 +73,7 @@ - + @@ -83,7 +83,7 @@ - + @@ -93,7 +93,7 @@ - + @@ -102,7 +102,7 @@ - + diff --git a/test/system/run_noresm_tests.sh b/test/system/run_noresm_tests.sh new file mode 100755 index 0000000000..887f641ef0 --- /dev/null +++ b/test/system/run_noresm_tests.sh @@ -0,0 +1,402 @@ +#! /bin/bash + +## +## General syntax help function +## Usage: help +## +help () { + local hname="Usage: $( basename ${0} )" + local hprefix="$( echo ${hname} | tr '[!-~]' ' ' )" + echo "${hname} --root " + echo "${hprefix} [ --project ]" + echo "${hprefix} [ --compare ]" + echo "${hprefix} [ --compiler ]" + echo "${hprefix} [ --generate ]" + echo "Less common options" + echo "${hprefix} [ --rerun ]" + echo "${hprefix} [ --baselinedir ]" + echo "${hprefix} [ --testdir ]" + echo "${hprefix} [ --clean ]" + echo "${hprefix} [ --cleanall ]" + echo "${hprefix} [ --dryrun ]" + echo "${hprefix} [ --rm-testdir ]" + exit $1 +} + +perr() { +## +## Output an error message <$2> if error code <$1> is non-zero +## + if [ $1 -ne 0 ]; then + echo -e "\nERROR ${1}: ${2}\n" + exit $1 + fi +} + +lastCamTag () { + local taglist + local tag + local currdir + currdir="$( pwd -P 2> /dev/null )" + if [ -d "${CAM_ROOT}/components/cam" ]; then + cd ${CAM_ROOT}/components/cam + elif [ -d "${CAM_ROOT}/models/atm/cam" ]; then + perr 1 "lastCamTag does not work with old directory structure" + elif [ -d "${CAM_ROOT}/test/system" ]; then + cd ${CAM_ROOT} + else + perr 1 "Cannot find CAM source at, \"${CAM_ROOT}\"" + fi + tag="$( git describe | sed -e 's/[-][0-9]*[-][0-9a-g]*$//' )" + cd ${currdir} + echo $tag +} + +extract_testtype () { + # extract_testtype testdir + local IFS + local dcomps + local ftype + local addtype + IFS=. + dcomps=(${1}) + shift + ftype="${dcomps[1]}" + addtype="yes" + for item in ${@}; do + if [ "${item}" == "${ftype}" ]; then + addtype="no" + fi + done + if [ "${addtype}" == "yes" ]; then + echo "${ftype}" + else + echo "" + fi +} + +clean_testdir () { + # clean_testdir CAM_TESTDIR cleanall [] + local cam_tdir # $1 + local cleanall # $2 + local cscmd + local ctest + local currdir + local test_id # $3 + local testdir + local tname + cam_tdir="${1}" + cleanall="${2}" + test_id="${3}" + currdir="$( pwd -P )" + # To clean tests, we need to find the proper test directory + if [ -n "${scratch}" -a -n "${test_id}" ]; then + testdir="${scratch}/${test_id}" + cscmd="${testdir}/cs.status.${test_id}" + if [ -f "${cscmd}" ]; then + fails="$( ${cscmd} | grep Overall | grep FAIL | cut -d' ' -f3 )" + else + fails="" + fi + for tname in ${fails}; do + ctest="${tname}.GC.${test_id}" + if [ "${cleanall}" == "yes" ]; then + rm -rf ${testdir}/${ctest} + else + cd ${testdir}/${ctest} + perr $? "Cannot enter ctest dir = '${testdir}/${ctest}'" + rm -f TestStatus* run/*.log* + rm -rf bld + ./.case.test --reset + ./case.setup --reset + fi + cd ${currdir} + done + fi + # Clean up old log files + # Do not remove the aux_cam_noresm_xxx files as those are reused. + rm -rf ${cam_tdir}/cime-tests.o* +} + +################################################## +## +## Beginning of script +## +################################################## + +# Make sure we are starting off in a valid directory +if [ ! -d "." ]; then + perr 1 "The current directory does not exist!!" +fi + +if [ -z "${currdir}" ]; then + currdir="$( pwd -P )" +fi +if [ -z "${scriptdir}" ]; then + scriptdir="$( cd $( dirname $0 ); pwd -P )" + perr $? "Line ${LINENO}: Cannot enter scriptdir = '${scriptdir}'" +fi + +# By default, let the machine set the baseline root +BL_ROOT="" +# Default baseline version for compare will be latest tag (from git describe) +bl_version="" +# Let machine set default compiler +CAM_FC="" +# No default for CAM_ROOT but env value allowed +CAM_ROOT=${CAM_ROOT:-""} +# Clean all test status +cleanall="no" +# Clean failures for successful rebuild +doclean="no" +# Dry run, print out the command but do not run it. +dryrun="no" +# Generate new baselines with this tag +new_tag="" +# Most test machines require a project code +project="" +# Remove test directory before beginning +removeall="no" +# Rerun an existing aux_cam_noresm suite +rerun="" + +## Process our input arguments +while [ $# -gt 0 ]; do + case $1 in + --h | -h | --help | -help) + help 0 + ;; + --baselinedir) + if [ ! -d "${2}" ]; then + perr 1 "Baseline root (${2}) must be an existing directory" + fi + BL_ROOT=${2} + shift + ;; + --compare) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a CAM tag for baseline tests" + fi + bl_version=${2} + shift + ;; + --clean) + doclean="yes" + ;; + --cleanall) + cleanall="yes" + ;; + --compiler) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a compiler name (e.g., INTEL)" + fi + CAM_FC="${2^^}" + shift + ;; + --dryrun) + dryrun="yes" + ;; + --generate) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a new CAM tag name" + fi + new_tag=${2} + shift + ;; + --project) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a project or accounting code" + fi + project=${2} + shift + ;; + --rerun) + rerun="--rerun-cesm" + ;; + --rm-testdir) + removeall="yes" + ;; + --root | -root) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a CAM root directory" + fi +# We need to make sure that the install directory is a full path +# First, we see if it looks like a full path (not sure Windows will like this) + case $1 in + /*) + CAM_ROOT=$2 + ;; + *) + if [ ! -d "${2}" ]; then + perr 1 "CAM root (${2}) must be an existing directory" + fi + CAM_ROOT="$( cd $2; pwd -P )" + perr $? "CAM root must exist" + esac + if [ ! -d "${CAM_ROOT}" ]; then + perr 1 "The specified CAM directory, \"${2}\", does not exist." + exit 1 + fi + shift + ;; + --testdir | -testdir) + if [ $# -lt 2 ]; then + perr 1 "${1} requires a test directory name" + fi + CAM_TESTDIR=${2} + shift + ;; + *) + perr 1 "Unrecognized option, \"${1}\"" + ;; + esac + shift +done + +if [ ! -d "${CAM_ROOT}" ]; then + perr 1 "Must specify CAM root with --root switch" +fi + +#################################### +## +## Load machine-dependent settings +## +#################################### + +if [ -z "${HOST}" -o "${HOST:0:5}" == "login" ]; then + export HOST="$( hostname )" +fi +if [ "$( echo ${HOST} | sed -e 's/^[^.]*[.]//' -e 's/[.].*$//' )" == "betzy" ]; then + machname="betzy" + CESMDATAROOT="/cluster/shared/noresm/inputdata" + if [ -z "${BL_ROOT}" ]; then + BL_ROOT="/cluster/shared/noresm/noresm_baselines" + fi + if [ -z "${CAM_FC}" ]; then + export CAM_FC="intel" + fi + scratch="/cluster/work/users/${USER}" +else + echo "ERROR: Unsupported host, \"${HOST}\"." + exit 3 +fi + +## We are running NorESM but that is not yet +## installed in the system as a separate model option. +export CIME_MODEL=cesm + +## If a baseline tag was specified, make sure it exists +if [ -n "${bl_version}" ]; then + if [ ! -d "${BL_ROOT}/${bl_version}" ]; then + perr "Baseline directory, '${BL_ROOT}/${bl_version}', must exist" + fi +fi + +## Create a test directory if necessary +if [ -z "${CAM_TESTDIR}" ]; then + CAM_TESTDIR="${scratch}/camtest_${CAM_FC,,}" +fi + +# Find the test_id if this is a rerun +if [ -n "${rerun}" ]; then + # We are doing a rerun, find the last test directory + # We should be in the place to look + logfile="$( ls ${CAM_TESTDIR}/aux_cam_noresm_${CAM_FC,,}_*.log | tail -n 1 )" + scriptdir=$( cd $( dirname ${0}); pwd -P ) + perr $? "Cannot enter old run dir = '${scriptdir}'" + test_info="$( ${scriptdir}/findTestInfo.py ${logfile} )" + test_tokens=( $test_info ) + if [ ${#test_tokens[@]} -eq 2 ]; then + CAM_TESTDIR=${test_tokens[0]} + test_id=${test_tokens[1]} + else + perr 1 "Cannot find test info, \"${test_info}\"" + fi + export CAM_TESTDIR + rerun="${rerun} ${test_id}" +else + test_id="" +fi + +# If this is a rerun, we may have to do a clean +if [ -n "${CAM_TESTDIR}" ]; then + if [ -d "${CAM_TESTDIR}" ]; then + if [ "${removeall}" == "yes" ]; then + if [ "${dryrun}" == "yes" ]; then + echo "Dryrun: Remove old test dir, '${CAM_TESTDIR}'" + else + rm -rf "${CAM_TESTDIR}" + fi + elif [ "${doclean}" == "yes" -o "${cleanall}" == "yes" ]; then + if [ "${dryrun}" == "yes" ]; then + echo "Dryrun: Clean test dir, '${CAM_TESTDIR}', test ID = '${test_id}'" + else + clean_testdir "${CAM_TESTDIR}" "${cleanall}" "${test_id}" + fi + fi + fi + if [ "${dryrun}" != "yes" ]; then + if [ ! -d "${CAM_TESTDIR}" ]; then + mkdir -p $CAM_TESTDIR + fi + CAM_TESTDIR="$( cd ${CAM_TESTDIR}; pwd -P )" + perr $? "Line ${LINENO}: Cannot enter CAM_TESTDIR = '${CAM_TESTDIR}'" + fi +fi + +# Export variables which might have been changed +export CAM_ROOT +if [ -d "${CAM_ROOT}/components/cam/test/system" ]; then + export CAM_STEST="${CAM_ROOT}/components/cam/test/system" +elif [ -d "${CAM_ROOT}/models/atm/cam/test/system" ]; then + export CAM_STEST="${CAM_ROOT}/models/atm/cam/test/system" +elif [ -d "${CAM_ROOT}/test/system" ]; then + export CAM_STEST="${CAM_ROOT}/test/system" +else + perr 1 "Cannot find system test directory from root, \"${CAM_ROOT}\"" +fi +export CAM_FC +if [ -n "${CAM_TESTDIR}" ]; then + export CAM_TESTDIR +fi + +#########################################3 + +if [ -n "${CAM_TESTDIR}" ]; then + if [ "${dryrun}" != "yes" ]; then + cd $CAM_TESTDIR + perr $? "Line ${LINENO}: Cannot enter CAM_TESTDIR = '${CAM_TESTDIR}'" + fi + echo "CAM_TESTDIR = ${CAM_TESTDIR}" +else + perr 1 "No value for CAM_TESTDIR" +fi +if [ -n "${CAM_ROOT}" ]; then + echo "CAM_ROOT = ${CAM_ROOT}" +fi +if [ -n "${CAM_FC}" ]; then + echo "Tests using the '${CAM_FC}' compiler" +fi + +command="${CAM_ROOT}/cime/scripts/create_test --xml-category aux_cam_noresm" +command="${command} --machine ${machname}" +command="${command} --test-root ${CAM_TESTDIR} --output-root ${CAM_TESTDIR}" +if [ -n "${project}" ]; then + command="${command} --project nn9560k" +fi +if [ -n "${bl_version}" -o -n "${new_tag}" ]; then + # We will generate and/or compare so enter a baseline root + command="${command} --baseline-root ${BL_ROOT}" +fi +if [ -n "${new_tag}" ]; then + command="${command} --test-id ${new_tag} --generate ${new_tag}" +fi +if [ -n "${bl_version}" ]; then + command="${command} --compare ${bl_version}" +fi + +if [ "${dryrun}" == "yes" ]; then + echo "Running: ${command}" +else + ${command} +fi diff --git a/test/system/test_driver.sh b/test/system/test_driver.sh deleted file mode 100755 index 964f647793..0000000000 --- a/test/system/test_driver.sh +++ /dev/null @@ -1,665 +0,0 @@ -#!/bin/sh -# -# test_driver.sh: driver for the testing of CAM with standalone scripts -# -# usage on hobart, izumi, leehill, cheyenne -# ./test_driver.sh -# -# **more details in the CAM testing user's guide, accessible -# from the CAM developers web page - -## -## General syntax help function -## Usage: help -## - -help () { - local hname="Usage: `basename ${0}` [ OPTION [ OPTION ... ] ]" - local hprefix="`echo ${hname} | tr '[!-~]' ' '`" - hprefix=" " - echo "${hname} " - echo "${hprefix} [ -b ] (support baseline scripts for cam5_2_12 and earlier)" - echo "${hprefix} [ -e ] (email summary to $USER)" - echo "${hprefix} [ -f ] (force batch submission -- avoids user prompt)" - echo "${hprefix} [ -h ] (displays this help message)" - echo "${hprefix} [ -i ] (interactive usage)" - echo "${hprefix} [ -j ] (number of jobs for gmake)" - echo "${hprefix} [ --baseline-dir ] (directory for saving baselines of cime tests)" - echo "${hprefix} [ --no-baseline] (baselines of cime tests are not saved)" - echo "${hprefix} [ --xml-driver ] (mct or nuopc)" - echo "${hprefix} [ --cesm ] (default aux_cam)" - echo "${hprefix} [ --rerun-cesm ] (rerun the cesm tests with the --use-existing-flag)" - echo "${hprefix} [ --namelists-only ] (Only perform namelist actions for tests. Incompatible with --rerun-cesm.)" - echo "${hprefix} [ --batch ] (Allow cime tests to run in parallel.)" - echo "" - echo "${hprefix} **pass environment variables by preceding above commands with:" - echo "${hprefix} 'env var1=setting var2=setting '" - echo "" - echo "Supported ENVIRONMENT variables" - echo "BL_TESTDIR: Default = none (used to set baseline compare dir)" - echo "CAM_ACCOUNT: Default = none" - echo "CAM_BATCHQ: Default = machine dependent" - echo "CAM_FC: Default = machine dependent" - echo "CAM_INPUT_TESTS: Default = tests_pretag_[_]" - echo "CAM_RESTART_TASKS: Default = 64" - echo "CAM_RETAIN_FILES: Default = FALSE" - echo "CAM_ROOT: Default = set relative to CAM_SCRIPTDIR" - echo "NB: If script is not called as ./`basename ${0}`, CAM_ROOT must be specified" - echo "CAM_SCRIPTDIR: Default = " - echo "CAM_TAG: Default = none (used to set CESM baseline dir)" - echo "CAM_TASKS: Default = (depends on system)" - echo "CAM_TESTDIR: Default = /test-driver." - echo "" - echo "Less common ENVIRONMENT variables" - echo "CALDERA_BATCHQ: Default = caldera" - echo "CAM_RBOPTIONS: Default = build_only" - echo "CAM_SOFF: Default = none (stop of first test fail if TRUE)" - echo "CIME_MODEL: Default = none (should be set to cesm)" - echo "EMAIL: Default = $USER@ucar.edu" - echo "SUMMARY_FILE: Default = `pwd -P`/cam_test_summaries}" - exit $1 -} - -## -## Error output function (should be handed a string) -## -perr() { - echo -e "\nERROR: ${@}\n" - help 1 -} - -## These variables may be overridden from the user's environment -EMAIL=${EMAIL:-"${USER}@ucar.edu"} - -# These variables may be modified by script switches (./test_driver.sh -h) -cam_email_summary=false -cesm_test_suite="aux_cam" -force=false -gmake_j=0 -interactive=false -use_existing='' -namelists_only=false -batch=false - -# Understand where we are and where the CAM root and CIME reside -if [ -n "${CAM_ROOT}" ]; then - # The user specified a CAM_ROOT, make sure it exists and that this - # script was called from its test directory. - if [ -d "${CAM_ROOT}" ]; then - test_dir="${CAM_ROOT}/test/system" - if [ -f "${test_dir}/test_driver.sh" ]; then - # Check against this script. - script_dir="$( cd $( dirname $0 ); pwd -P )" - script_file="${script_dir}/test_driver.sh" - if [ "${test_dir}/test_driver.sh" != "${script_file}" ]; then - perr "CAM_ROOT test dir is ${test_dir} but script is ${script_file}" - fi # Else, everything is fine - else - perr "No test_driver.sh found in ${test_dir}" - fi - else - perr "CAM_ROOT, '${CAM_ROOT}', does not exist" - fi -else - # The user did not specify CAM_ROOT, find it relative to this script - test_dir="$( dirname $0 )" - export CAM_ROOT="$(dirname $(dirname $( cd ${test_dir}; pwd -P )))" -fi -# Now, find CIME_ROOT, first try a CAM standalone checkout -if [ -d "${CAM_ROOT}/cime/scripts" ]; then - export CIME_ROOT="${CAM_ROOT}/cime" -else - export CIME_ROOT="$( dirname $( dirname ${CAM_ROOT} ) )/cime" - if [ ! -d "${CIME_ROOT}/scripts" ]; then - perr "No CIME found from CAM_ROOT = '${CAM_ROOT}'" - fi -fi - -# Initialize variables which may not be set -submit_script_cime='' - -while [ "${1:0:1}" == "-" ]; do - case $1 in - - --baseline-dir ) - if [ $# -lt 2 ]; then - perr "${1} requires a directory name)" - fi - baseline_dir="${2}" - shift - ;; - - --no-baseline ) no_baseline=false - ;; - - -b ) export CAM_BASEBACK="YES" - ;; - - --cesm ) - if [ $# -lt 2 ]; then - perr "${1} requires a CESM test name or test suite name (e.g., aux_cam)" - fi - if [ "${2:0:1}" == "-" ]; then - perr "Invalid CESM test name, '${2}'" - fi - cesm_test_suite="${2}" - shift - ;; - - -e ) cam_email_summary=true - ;; - - -f ) force=true - if $interactive ; then - echo "test_driver.sh: FATAL ERROR: -i and -f were set" - exit 1 - fi - ;; - - -h | --help ) - help 0 - ;; - - -i ) interactive=true - if $force ; then - echo "test_driver.sh: FATAL ERROR: -i and -f were set" - exit 1 - fi - ;; - - -j ) shift; gmake_j=$1 - ;; - - --rerun-cesm ) - if [ $# -lt 2 ]; then - perr "${1} requires a test_id from a previous run)" - fi - use_existing="${2}" - shift - if $namelists_only ; then - echo "test_driver.sh: FATAL ERROR: --rerun-cesm and --namelists-only were set" - exit 1 - fi - ;; - - --xml-driver ) - if [ $# -lt 2 ]; then - perr "${1} specify mct or nuopc)" - fi - xml_driver="${2}" - shift - ;; - - --namelists-only ) - namelists_only=true - if [ "${use_existing}" != "" ]; then - echo "test_driver.sh: FATAL ERROR: --namelists-only and --rerun-cesm were set" - exit 1 - fi - ;; - - --batch ) - batch=true - ;; - - esac - shift -done - -# Currently, we don't support non-options, should we? -if [ $# -gt 0 ]; then - perr "Unrecognized arguments: '$*'" -fi - -#will attach timestamp onto end of script name to prevent overwriting -start_date="`date --iso-8601=seconds`" -cur_time=`date '+%H%M%S'` -date_str="`date '+%Y%m%d%H%M%S'`" - -hostname=`hostname` - -case $hostname in - - ##cheyenne - ch* | r* ) - submit_script_cime="`pwd -P`/test_driver_cheyenne_cime_${cur_time}.sh" - - if [ -z "$CAM_ACCOUNT" ]; then - echo "ERROR: Must set the environment variable CAM_ACCOUNT" - exit 2 - fi - - if [ -z "$CAM_BATCHQ" ]; then - export CAM_BATCHQ="regular" - fi - - # wallclock for run job - wallclock_limit="5:00:00" - - if [ $gmake_j = 0 ]; then - gmake_j=36 - fi - - # run tests on 2 nodes using 18 tasks/node, 2 threads/task - CAM_TASKS=36 - CAM_THREADS=2 - - # change parallel configuration on 2 nodes using 32 tasks, 1 threads/task - CAM_RESTART_TASKS=32 - CAM_RESTART_THREADS=1 - - mach_workspace="/glade/scratch" - - # Check for CESM baseline directory - if [ -n "${BL_TESTDIR}" ] && [ ! -d "${BL_TESTDIR}" ]; then - echo "CESM_BASELINE ${BL_TESTDIR} not found. Check BL_TESTDIR for correct tag name." - exit - fi - -#------------------------------------------- - -cat > ${submit_script_cime} << EOF -#!/bin/bash -# -#PBS -N cime-tests -#PBS -q $CAM_BATCHQ -#PBS -A $CAM_ACCOUNT -#PBS -l walltime=4:00:00 -#PBS -l select=1:ncpus=36:mpiprocs=36 -#PBS -j oe -#PBS -l inception=login - -EOF - -##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ - ;; - - ##hobart - hob* | h[[:digit:]]* ) - submit_script_cime="`pwd -P`/test_driver_hobart_cime_${cur_time}.sh" - export PATH=/cluster/torque/bin:${PATH} - - # Default setting is 12 hr in the long queue; the short queue only - # allows 1 hr runs. - wallclock_limit="12:00:00" - gmake_j=24 - if [ -z "$CAM_BATCHQ" ]; then - export CAM_BATCHQ="long" - elif [[ "$CAM_BATCHQ" == short ]]; then - wallclock_limit="1:00:00" - fi - - if [ $gmake_j = 0 ]; then - gmake_j=24 - fi - - if [ -z "$CAM_TASKS" ]; then - CAM_TASKS=24 - fi - if [ -z "$CAM_RESTART_TASKS" ]; then - CAM_RESTART_TASKS=$(( $CAM_TASKS / 2)) - fi - - mach_workspace="/scratch/cluster" - - # Check for CESM baseline directory - if [ -n "{$BL_TESTDIR}" ] && [ ! -d "${BL_TESTDIR}" ]; then - echo "CESM_BASELINE ${BL_TESTDIR} not found. Check BL_TESTDIR for correct tag name." - exit - fi - -#------------------------------------------- - -cat > ${submit_script_cime} << EOF -#!/bin/bash -# -# Name of the queue (CHANGE THIS if needed) -#PBS -q $CAM_BATCHQ -# Number of nodes (CHANGE THIS if needed) -#PBS -l walltime=$wallclock_limit,nodes=1:ppn=24 -# output file base name -#PBS -N cime-tests -# Put standard error and standard out in same file -#PBS -j oe -# Export all Environment variables -#PBS -V -# End of options - -EOF - -##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ - ;; - - ##izumi - izu* | i[[:digit:]]* ) - - submit_script_cime="`pwd -P`/test_driver_izumi_cime_${cur_time}.sh" - export PATH=/cluster/torque/bin:${PATH} - - # Default setting is 12 hr in the long queue; the short queue only - # allows 1 hr runs. - wallclock_limit="12:00:00" - gmake_j=24 - if [ -z "$CAM_BATCHQ" ]; then - export CAM_BATCHQ="long" - elif [[ "$CAM_BATCHQ" == short ]]; then - wallclock_limit="1:00:00" - fi - - if [ $gmake_j = 0 ]; then - gmake_j=24 - fi - - if [ -z "$CAM_TASKS" ]; then - CAM_TASKS=24 - fi - if [ -z "$CAM_RESTART_TASKS" ]; then - CAM_RESTART_TASKS=$(( $CAM_TASKS / 2)) - fi - - mach_workspace="/scratch/cluster" - - # Check for CESM baseline directory - if [ -n "{$BL_TESTDIR}" ] && [ ! -d "${BL_TESTDIR}" ]; then - echo "CESM_BASELINE ${BL_TESTDIR} not found. Check BL_TESTDIR for correct tag name." - exit - fi - -#------------------------------------------- - -cat > ${submit_script_cime} << EOF -#!/bin/bash -# -# Name of the queue (CHANGE THIS if needed) -#PBS -q $CAM_BATCHQ -# Number of nodes (CHANGE THIS if needed) -#PBS -l walltime=$wallclock_limit,nodes=1:ppn=24 -# output file base name -#PBS -N cime-tests -# Put standard error and standard out in same file -#PBS -j oe -# Export all Environment variables -#PBS -V -# End of options - -EOF - -##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ - ;; - - ##casper - casper* | crhtc* ) - submit_script_cime="`pwd -P`/test_driver_casper_cime_${cur_time}.sh" - - if [ -z "$CAM_ACCOUNT" ]; then - echo "ERROR: Must set the environment variable CAM_ACCOUNT" - exit 2 - fi - - if [ -z "$CAM_BATCHQ" ]; then - export CAM_BATCHQ="casper" - fi - - # wallclock for run job - wallclock_limit="00:59:00" - - if [ $gmake_j = 0 ]; then - gmake_j=36 - fi - - # run tests on 1 nodes using 18 tasks/node, 2 threads/task - CAM_TASKS=18 - CAM_THREADS=2 - - # change parallel configuration on 1 nodes using 32 tasks, 1 threads/task - CAM_RESTART_TASKS=32 - CAM_RESTART_THREADS=1 - - mach_workspace="/glade/scratch" - - # Check for CESM baseline directory - if [ -n "${BL_TESTDIR}" ] && [ ! -d "${BL_TESTDIR}" ]; then - echo "CESM_BASELINE ${BL_TESTDIR} not found. Check BL_TESTDIR for correct tag name." - exit - fi - -#------------------------------------------- - -cat > ${submit_script_cime} << EOF -#!/bin/bash -# -#PBS -N cime-tests -#PBS -q $CAM_BATCHQ -#PBS -A $CAM_ACCOUNT -#PBS -l walltime=2:00:00 -#PBS -l select=1:ncpus=36:mpiprocs=36:mem=300GB -#PBS -j oe -#PBS -V -EOF - -##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^ - ;; - - * ) echo "ERROR: machine $hostname not currently supported"; exit 1 ;; -esac - - -##vvvvvvvvvvvvvvvvvvvvvv start CAM aux test suite vvvvvvvvvvvvvvvvvvvvvvvvvvvv - -cesm_test_mach="" -comp="" -if [ "${hostname:0:4}" == "chey" ]; then - cesm_test_mach="cheyenne" -fi -if [ "${hostname:0:6}" == "hobart" ]; then - cesm_test_mach="hobart" -fi -if [ "${hostname:0:5}" == "izumi" ]; then - cesm_test_mach="izumi" -fi -if [ "${hostname:0:6}" == "casper" ] || [ "${hostname:0:5}" == "crhtc" ]; then - cesm_test_mach="casper" -fi -if [ -n "${CAM_FC}" ]; then - comp="_${CAM_FC,,}" -fi - -if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then - if [ "${hostname:0:5}" != "izumi" ]; then - module load python - fi - - - for cesm_test in ${cesm_test_suite}; do - testargs="--xml-category ${cesm_test} --xml-machine ${cesm_test_mach} --retry 2" - - if [ -n "${use_existing}" ]; then - test_id="${use_existing}" - else - test_id=${cesm_test}${comp}"_"${date_str} - fi - currdir="`pwd -P`" - logfile="${currdir}/${test_id}.log" - # Create an empty logfile so that other tasks can append to it - if [ -f "${logfile}" ]; then - rm -f ${logfile} - fi - touch ${logfile} - script_dir="${CIME_ROOT}/scripts" - if [ ! -d "${script_dir}" ]; then - echo "ERROR: CIME scripts dir not found at ${script_dir}" - exit 1 - fi - if [ ! -x "${script_dir}/create_test" ]; then - echo "ERROR: create_test script dir not found in ${script_dir}" - exit 1 - fi - - ## If this is a Nag test, run the r8 and git tests - if [ "${comp}" == "_nag" ]; then - sepstr="################################################################" - echo "${sepstr}" | tee -a ${logfile} - ark_file="/fs/cgd/csm/tools/addrealkind/addrealkind" - tr8_script="${CAM_ROOT}/test/system/TR8.sh" - export ADDREALKIND_EXE="${ark_file}"; ${tr8_script} | tee -a ${logfile} - res=${PIPESTATUS[0]} - if [ $res -eq 0 ]; then - echo "TR8 test PASS" | tee -a ${logfile} - else - echo "TR8 test FAIL, rc = $res" | tee -a ${logfile} - fi - echo "${sepstr}" | tee -a ${logfile} - ${CAM_ROOT}/test/system/TGIT.sh | tee -a ${logfile} - res=${PIPESTATUS[0]} - if [ $res -eq 0 ]; then - echo "TGIT test PASS" | tee -a ${logfile} - else - echo "TGIT test FAIL, rc = $res" | tee -a ${logfile} - fi - echo "${sepstr}" | tee -a ${logfile} - fi - - ## Setup CESM work directory - if [ "${hostname:0:6}" == "casper" ] || [ "${hostname:0:5}" == "crhtc" ]; then - ## Would fail to compile on Casper with long folder name - cesm_testdir=$mach_workspace/$LOGNAME/$cesm_test - else - cesm_testdir=$mach_workspace/$LOGNAME/$test_id - fi - - if [ -e ${cesm_testdir} ]; then - if [ -n "${use_existing}" ]; then - echo " Using existing tests in ${cesm_testdir}" - else - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! " - echo "!! ERROR: ${cesm_testdir} already exists and << --rerun-cesm >> was not specified " - echo "!! Either remove ${cesm_testdir} or specify << --rerun-cesm >> " - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! " - exit 1 - fi - else - mkdir $cesm_testdir - fi - - if [ -n "${CAM_FC}" ]; then - testargs="${testargs} --xml-compiler ${CAM_FC,,}" - else - testargs="${testargs} --xml-compiler intel" - fi - case $hostname in - # cheyenne - chey* | r* ) - testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" - ;; - # casper - casper* | crhtc* ) - testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" - ;; - *) - if $batch; then - testargs="${testargs} --queue ${CAM_BATCHQ} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" - else - testargs="${testargs} --test-root ${cesm_testdir} --output-root ${cesm_testdir}" - testargs="${testargs} --no-batch" - fi - esac - if [ -n "${CAM_ACCOUNT}" ]; then - testargs="${testargs} --project ${CAM_ACCOUNT}" - fi - testargs="${testargs} --test-id ${test_id}" - if [ -n "${BL_TESTDIR}" ]; then - testargs="${testargs} --compare ${BL_TESTDIR} " - fi - if [ -n "${use_existing}" ]; then - testargs="${testargs} --use-existing -o " - fi - if $namelists_only ; then - testargs="${testargs} --namelists-only " - fi - # Check for a change in BL_TESTDIR # - if [ -n "${BL_TESTDIR}" ] && [ "${use_existing}" != "" ]; then - #Check if BL_TESTDIR changed - cmd="query_testlists --xml-category $cesm_test --xml-machine ${cesm_test_mach}" - if [ -n "${CAM_FC}" ]; then - cmd="${cmd} --xml-compiler ${CAM_FC,,}" - else - cmd="${cmd} --xml-compiler intel" - fi - cmd="${CIME_ROOT}/scripts/"$cmd - cime_testlist=`$cmd` - for i in $(echo $cime_testlist | tr " " "\n") - do - if [[ $i =~ ${cesm_test_mach} ]]; then - orig_baseline=`cd $cesm_testdir/$i*$test_id && ./xmlquery BASELINE_NAME_CMP --value` - if [ $orig_baseline != ${BL_TESTDIR} ]; then - echo "Changing BL_TESTDIR for $i." - `cd $cesm_testdir/$i*$test_id && ./xmlchange BASELINE_NAME_CMP=$BL_TESTDIR` - if [[ $i == ERI* ]]; then #Need to do special stuff to get ERI to rerun with new baseline. - `cd $cesm_testdir/$i*$test_id && sed -i '/RUN/c\FAIL '$i' RUN' TestStatus` - result=`cd $cesm_testdir/$i*$test_id && pwd && ./.case.test --reset -s` - else - `cd $cesm_testdir/$i*$test_id && sed -i '/RUN/c\PEND '$i' RUN' TestStatus` - fi - else - echo "Checking for changed BL_TESTDIR for $i." - fi - fi - done - fi - - if [ "$no_baseline" != false ]; then - if [ -n "${baseline_dir}" ]; then - testargs="${testargs} --generate ${baseline_dir}" - else - testargs="${testargs} --generate ${cesm_testdir}/baselines" - fi - fi - - if [ -n "${xml_driver}" ]; then - testargs="${testargs} --xml-driver ${xml_driver}" - fi - - echo "" - echo "CESM test results will be in: ${cesm_testdir}" | tee -a ${logfile} - echo "Running ./create_test ${testargs}" | tee -a ${logfile} - - if [ "${hostname:0:2}" == "ch" ]; then - echo "cd ${script_dir}" >> ${submit_script_cime} - echo "module load python" >> ${submit_script_cime} - echo './create_test' ${testargs} >> ${submit_script_cime} - chmod u+x ${submit_script_cime} - qsub ${submit_script_cime} - fi - - if [ "${hostname:0:6}" == "hobart" ]; then - echo "cd ${script_dir}" >> ${submit_script_cime} - echo './create_test' ${testargs} >> ${submit_script_cime} - if [ "${submit_script}" != "${submit_script_cime}" ]; then - chmod u+x ${submit_script_cime} - qsub ${submit_script_cime} - fi - fi - - if [ "${hostname:0:5}" == "izumi" ]; then - echo "cd ${script_dir}" >> ${submit_script_cime} - echo './create_test' ${testargs} >> ${submit_script_cime} - if [ "${submit_script}" != "${submit_script_cime}" ]; then - chmod u+x ${submit_script_cime} - qsub ${submit_script_cime} - fi - fi - - if [ "${hostname:0:6}" == "casper" ] || [ "${hostname:0:5}" == "crhtc" ]; then - echo "cd ${script_dir}" >> ${submit_script_cime} - echo "module load python" >> ${submit_script_cime} - echo './create_test' ${testargs} >> ${submit_script_cime} - chmod u+x ${submit_script_cime} - qsub ${submit_script_cime} - fi - - done -fi - -##^^^^^^^^^^^^^^^^^^^^^^ start CAM aux test suite ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -exit 0