Skip to content

Commit

Permalink
Still WIP, add UFS resources as a template
Browse files Browse the repository at this point in the history
  • Loading branch information
DavidHuber-NOAA committed May 17, 2024
1 parent 8238d33 commit d300801
Show file tree
Hide file tree
Showing 6 changed files with 664 additions and 712 deletions.
8 changes: 4 additions & 4 deletions parm/config/gfs/config.atmanl
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ fi
export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"

export layout_x_atmanl=@LAYOUT_X_ATMANL@
export layout_y_atmanl=@LAYOUT_Y_ATMANL@
export layout_x
export layout_y

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@
export io_layout_x
export io_layout_y

export JEDIEXE=${EXECgfs}/gdas.x

Expand Down
133 changes: 0 additions & 133 deletions parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -588,150 +588,17 @@ case ${step} in
"fcst" | "efcs")
export is_exclusive=True

if [[ "${step}" == "fcst" ]]; then
_CDUMP_LIST=${CDUMP:-"gdas gfs"}
elif [[ "${step}" == "efcs" ]]; then
_CDUMP_LIST=${CDUMP:-"enkfgdas enkfgfs"}
fi

# During workflow creation, we need resources for all CDUMPs and CDUMP is undefined
for _CDUMP in ${_CDUMP_LIST}; do
if [[ "${_CDUMP}" =~ "gfs" ]]; then
export layout_x=${layout_x_gfs}
export layout_y=${layout_y_gfs}
export WRITE_GROUP=${WRITE_GROUP_GFS}
export WRTTASK_PER_GROUP_PER_THREAD=${WRTTASK_PER_GROUP_PER_THREAD_GFS}
ntasks_fv3=${ntasks_fv3_gfs}
ntasks_quilt=${ntasks_quilt_gfs}
nthreads_fv3=${nthreads_fv3_gfs}
nthreads_ufs=${nthreads_ufs_gfs}
fi

# Determine if using ESMF-managed threading or traditional threading
# If using traditional threading, set them to 1
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
export UFS_THREADS=1
else # traditional threading
export UFS_THREADS=${nthreads_ufs:-1}
nthreads_fv3=1
nthreads_mediator=1
[[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
[[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
[[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
fi

if (( ntiles > 6 )); then
export layout_x_nest=${layout_x_nest:-10}
export layout_y_nest=${layout_y_nest:-10}
export npx_nest=${npx_nest:-1441}
export npy_nest=${npy_nest:-961}
fi

# PETS for the atmosphere dycore
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"

# PETS for quilting
if [[ "${QUILTING:-}" == ".true." ]]; then
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
export WRTTASK_PER_GROUP
else
QUILTPETS=0
fi
echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"

# Total PETS for the atmosphere component
ATMTHREADS=${nthreads_fv3}
(( ATMPETS = FV3PETS + QUILTPETS ))
export ATMPETS ATMTHREADS
echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"

# Total PETS for the coupled model (starting w/ the atmosphere)
NTASKS_TOT=${ATMPETS}

# The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
# However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
# See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
# TODO: Update reference when moved to ufs-weather-model RTD
MEDTHREADS=${nthreads_mediator:-1}
MEDPETS=${MEDPETS:-${FV3PETS}}
(( "${MEDPETS}" > 300 )) && MEDPETS=300
export MEDPETS MEDTHREADS
echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"

CHMPETS=0; CHMTHREADS=0
if [[ "${DO_AERO}" == "YES" ]]; then
# GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
(( CHMTHREADS = ATMTHREADS ))
(( CHMPETS = FV3PETS ))
# Do not add to NTASKS_TOT
echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})"
fi
export CHMPETS CHMTHREADS

WAVPETS=0; WAVTHREADS=0
if [[ "${DO_WAVE}" == "YES" ]]; then
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
(( WAVTHREADS = nthreads_ww3 ))
echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
(( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
fi
export WAVPETS WAVTHREADS

OCNPETS=0; OCNTHREADS=0
if [[ "${DO_OCN}" == "YES" ]]; then
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
(( OCNTHREADS = nthreads_mom6 ))
echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
(( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
fi
export OCNPETS OCNTHREADS

ICEPETS=0; ICETHREADS=0
if [[ "${DO_ICE}" == "YES" ]]; then
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
(( ICETHREADS = nthreads_cice6 ))
echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
(( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
fi
export ICEPETS ICETHREADS

echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}"

if [[ "${_CDUMP}" =~ "gfs" ]]; then
declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
declare -x "nth_${step}_gfs"="${UFS_THREADS}"
declare -x "npe_node_${step}_gfs"="${npe_node_max}"
else
declare -x "npe_${step}"="${NTASKS_TOT}"
declare -x "nth_${step}"="${UFS_THREADS}"
declare -x "npe_node_${step}"="${npe_node_max}"
fi

done

case "${CASE}" in
"C48" | "C96" | "C192")
declare -x "wtime_${step}"="00:20:00"
declare -x "wtime_${step}_gfs"="03:00:00"
;;
"C384")
declare -x "wtime_${step}"="00:30:00"
declare -x "wtime_${step}_gfs"="06:00:00"
;;
"C768" | "C1152")
declare -x "wtime_${step}"="00:30:00"
declare -x "wtime_${step}_gfs"="06:00:00"
;;
*)
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
exit 4
;;
esac

unset _CDUMP _CDUMP_LIST
unset NTASKS_TOT
;;

"oceanice_products")
Expand Down
7 changes: 5 additions & 2 deletions parm/config/gfs/config.snowanl
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

echo "BEGIN: config.snowanl"

export layout_x
export layout_y

# Get task specific resources
source "${EXPDIR}/config.resources" snowanl

Expand All @@ -24,7 +27,7 @@ export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/apply_incr_nml.j2"

export JEDI_FIX_YAML="${PARMgfs}/gdas/snow_jedi_fix.yaml.j2"

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@
export io_layout_x
export io_layout_y

echo "END: config.snowanl"
Loading

0 comments on commit d300801

Please sign in to comment.