Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/opt resources #12

Closed
wants to merge 28 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
b84b3b3
WIP resource yaml
DavidHuber-NOAA May 14, 2024
8238d33
Merge remote-tracking branch 'origin/develop' into feature/opt_resources
DavidHuber-NOAA May 15, 2024
d300801
Still WIP, add UFS resources as a template
DavidHuber-NOAA May 17, 2024
16d841e
Flesh out resources yaml
DavidHuber-NOAA May 23, 2024
13f5105
Prepend resource options to configs
DavidHuber-NOAA May 24, 2024
e8f6d44
Flatten resources yaml structure
DavidHuber-NOAA May 24, 2024
58a10ab
Remove (should be) unnecessary resource definitsion from env files
DavidHuber-NOAA May 24, 2024
941b30f
Fixed PEP issues
DavidHuber-NOAA May 28, 2024
090b2d3
Improve resource yaml naming and tabbing
DavidHuber-NOAA May 28, 2024
528b55f
Improve tabbing, add S4 overrides, create failsafe defaults for CASE-…
DavidHuber-NOAA May 28, 2024
656f4be
Renamed/moved yamls
DavidHuber-NOAA May 31, 2024
b67c549
Create a dynamic resource calculator
DavidHuber-NOAA May 31, 2024
cdf88a3
Add machine yamls
DavidHuber-NOAA May 31, 2024
a931d8b
Add Hercules RAM info
DavidHuber-NOAA May 31, 2024
7c59b9c
Improve resource yamls
DavidHuber-NOAA May 31, 2024
e97dd9f
Correct the name of detect_py_ver
DavidHuber-NOAA May 31, 2024
dfe8c38
Update S4 resources
DavidHuber-NOAA May 31, 2024
7a18460
Remove resource calculations from applications.py, use ResourceConfig…
DavidHuber-NOAA May 31, 2024
df1af59
Fix syntax/style
DavidHuber-NOAA Jun 3, 2024
228cec4
Assign default and max memory cases
DavidHuber-NOAA Jun 3, 2024
84c84d2
Minimize jinja use in resources yaml
DavidHuber-NOAA Jun 4, 2024
11ae211
Initial crack at reworking fcst/efcs resources.
DavidHuber-NOAA Jun 6, 2024
39ede7c
Revert env files for now
DavidHuber-NOAA Jun 6, 2024
9244d3d
Merge remote-tracking branch 'origin/develop' into feature/opt_resources
DavidHuber-NOAA Jun 6, 2024
ffdcd28
Reverted more changes
DavidHuber-NOAA Jun 6, 2024
0b4c997
Fix typo
DavidHuber-NOAA Jun 6, 2024
c33c11c
Removed test file
DavidHuber-NOAA Jun 6, 2024
2d3b6a0
Final update, add resources for nested and GFS
DavidHuber-NOAA Jun 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 0 additions & 133 deletions parm/config/gfs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -618,150 +618,17 @@ case ${step} in
"fcst" | "efcs")
export is_exclusive=True

if [[ "${step}" == "fcst" ]]; then
_CDUMP_LIST=${CDUMP:-"gdas gfs"}
elif [[ "${step}" == "efcs" ]]; then
_CDUMP_LIST=${CDUMP:-"enkfgdas enkfgfs"}
fi

# During workflow creation, we need resources for all CDUMPs and CDUMP is undefined
for _CDUMP in ${_CDUMP_LIST}; do
if [[ "${_CDUMP}" =~ "gfs" ]]; then
export layout_x=${layout_x_gfs}
export layout_y=${layout_y_gfs}
export WRITE_GROUP=${WRITE_GROUP_GFS}
export WRTTASK_PER_GROUP_PER_THREAD=${WRTTASK_PER_GROUP_PER_THREAD_GFS}
ntasks_fv3=${ntasks_fv3_gfs}
ntasks_quilt=${ntasks_quilt_gfs}
nthreads_fv3=${nthreads_fv3_gfs}
nthreads_ufs=${nthreads_ufs_gfs}
fi

# Determine if using ESMF-managed threading or traditional threading
# If using traditional threading, set them to 1
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
export UFS_THREADS=1
else # traditional threading
export UFS_THREADS=${nthreads_ufs:-1}
nthreads_fv3=1
nthreads_mediator=1
[[ "${DO_WAVE}" == "YES" ]] && nthreads_ww3=1
[[ "${DO_OCN}" == "YES" ]] && nthreads_mom6=1
[[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
fi

if (( ntiles > 6 )); then
export layout_x_nest=${layout_x_nest:-10}
export layout_y_nest=${layout_y_nest:-10}
export npx_nest=${npx_nest:-1441}
export npy_nest=${npy_nest:-961}
fi

# PETS for the atmosphere dycore
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"

# PETS for quilting
if [[ "${QUILTING:-}" == ".true." ]]; then
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
export WRTTASK_PER_GROUP
else
QUILTPETS=0
fi
echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"

# Total PETS for the atmosphere component
ATMTHREADS=${nthreads_fv3}
(( ATMPETS = FV3PETS + QUILTPETS ))
export ATMPETS ATMTHREADS
echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"

# Total PETS for the coupled model (starting w/ the atmosphere)
NTASKS_TOT=${ATMPETS}

# The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
# However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
# See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
# TODO: Update reference when moved to ufs-weather-model RTD
MEDTHREADS=${nthreads_mediator:-1}
MEDPETS=${MEDPETS:-${FV3PETS}}
(( "${MEDPETS}" > 300 )) && MEDPETS=300
export MEDPETS MEDTHREADS
echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"

CHMPETS=0; CHMTHREADS=0
if [[ "${DO_AERO}" == "YES" ]]; then
# GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
(( CHMTHREADS = ATMTHREADS ))
(( CHMPETS = FV3PETS ))
# Do not add to NTASKS_TOT
echo "GOCART using (threads, PETS) = (${CHMTHREADS}, ${CHMPETS})"
fi
export CHMPETS CHMTHREADS

WAVPETS=0; WAVTHREADS=0
if [[ "${DO_WAVE}" == "YES" ]]; then
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
(( WAVTHREADS = nthreads_ww3 ))
echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
(( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
fi
export WAVPETS WAVTHREADS

OCNPETS=0; OCNTHREADS=0
if [[ "${DO_OCN}" == "YES" ]]; then
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
(( OCNTHREADS = nthreads_mom6 ))
echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
(( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
fi
export OCNPETS OCNTHREADS

ICEPETS=0; ICETHREADS=0
if [[ "${DO_ICE}" == "YES" ]]; then
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
(( ICETHREADS = nthreads_cice6 ))
echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
(( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
fi
export ICEPETS ICETHREADS

echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}"

if [[ "${_CDUMP}" =~ "gfs" ]]; then
declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
declare -x "nth_${step}_gfs"="${UFS_THREADS}"
declare -x "npe_node_${step}_gfs"="${npe_node_max}"
else
declare -x "npe_${step}"="${NTASKS_TOT}"
declare -x "nth_${step}"="${UFS_THREADS}"
declare -x "npe_node_${step}"="${npe_node_max}"
fi

done

case "${CASE}" in
"C48" | "C96" | "C192")
declare -x "wtime_${step}"="00:20:00"
declare -x "wtime_${step}_gfs"="03:00:00"
;;
"C384")
declare -x "wtime_${step}"="00:30:00"
declare -x "wtime_${step}_gfs"="06:00:00"
;;
"C768" | "C1152")
declare -x "wtime_${step}"="00:30:00"
declare -x "wtime_${step}_gfs"="06:00:00"
;;
*)
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
exit 4
;;
esac

unset _CDUMP _CDUMP_LIST
unset NTASKS_TOT
;;

"oceanice_products")
Expand Down
Loading
Loading