From 0a0d6984e094d2a551d5d8331102142e7db13fc2 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 14 Nov 2023 14:08:26 -0500 Subject: [PATCH] Enable warm starting of the model for forecast-only mode (#2031) This PR: - adds an option to warm|cold start the ufs-weather-model when setting up an experiment for GFS/GEFS - adds the capability to stage FV3 restarts when the atmosphere is warm started from FV3 restarts. This would be used in the GEFS free-forecast for part of the reforecast capability. - If a coupled model is warm started for the atmosphere, also check if mediator restarts are present. If so, stage mediator restarts to allow restarting the coupled model. Fixes #2016 --- ci/cases/pr/C48_S2SA_gefs.yaml | 2 +- parm/config/gefs/config.base.emc.dyn | 7 +- scripts/exglobal_stage_ic.sh | 102 +++++++++++++++++++-------- workflow/setup_expt.py | 3 +- 4 files changed, 82 insertions(+), 32 deletions(-) diff --git a/ci/cases/pr/C48_S2SA_gefs.yaml b/ci/cases/pr/C48_S2SA_gefs.yaml index 927a979b13..2abbf0043e 100644 --- a/ci/cases/pr/C48_S2SA_gefs.yaml +++ b/ci/cases/pr/C48_S2SA_gefs.yaml @@ -9,9 +9,9 @@ arguments: resens: 48 nens: 2 gfs_cyc: 1 + start: cold comrot: {{ 'RUNTESTS' | getenv }}/COMROT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48C48mx500 idate: 2021032312 edate: 2021032312 yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn index b62c921ed0..04fc396e5a 100644 --- a/parm/config/gefs/config.base.emc.dyn +++ b/parm/config/gefs/config.base.emc.dyn @@ -282,11 +282,16 @@ export NMEM_ENS=@NMEM_ENS@ export ENSMEM="000" export MEMDIR="mem${ENSMEM}" +export DOIAU="NO" # While we are not doing IAU, we may want to warm start w/ IAU in the future # Check if cycle is cold starting if [[ "${EXP_WARM_START}" = ".false." ]]; then - export IAU_FHROT=0 + export IAU_FHROT=0 else + if [[ "${DOIAU}" = "YES" ]]; then export IAU_FHROT=3 + else + export IAU_FHROT=0 + fi fi # turned on nsst in anal and/or fcst steps, and turn off rtgsst diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index 53042c7e45..6276255167 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -9,9 +9,9 @@ gPDY="${GDATE:0:8}" gcyc="${GDATE:8:2}" MEMDIR_ARRAY=() -if [[ "${RUN}" == "gefs" ]]; then +if [[ "${RUN:-}" = "gefs" ]]; then # Populate the member_dirs array based on the value of NMEM_ENS - for ((ii = 0; ii <= "${NMEM_ENS}"; ii++)); do + for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") done else @@ -27,35 +27,58 @@ error_message() { ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - # Stage the FV3 initial conditions to ROTDIR (cold start) - YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT - [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}" - src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc" - tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - for ftype in gfs_data sfc_data; do - for ((tt = 1; tt <= 6; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" - tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" + + # Stage atmosphere initial conditions to ROTDIR + if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then + # Stage the FV3 restarts to ROTDIR (warm start) + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}" + for ftype in coupler.res fv_core.res.nc; do + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}" + tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" err=$((err + rc)) done - done + for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do + for ((tt = 1; tt <= 6; tt++)); do + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" + tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" + ${NCP} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + done + done + else + # Stage the FV3 cold-start initial conditions to ROTDIR + YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT + [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc" + tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc" + ${NCP} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + for ftype in gfs_data sfc_data; do + for ((tt = 1; tt <= 6; tt++)); do + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" + tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" + ${NCP} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + done + done + fi # Stage ocean initial conditions to ROTDIR (warm start) if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART - [[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}" - src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc" - tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc" + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL + [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}" + src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc" + tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res.nc" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -66,7 +89,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do ;; "025" ) for nn in $(seq 1 3); do - src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc" + src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc" tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res_${nn}.nc" ${NCP} "${src}" "${tgt}" rc=$? @@ -80,13 +103,33 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do err=$((err + rc)) ;; esac + + # TODO: Do mediator restarts exists in a ATMW configuration? + # TODO: No mediator is presumably involved in an ATMA configuration + if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then + # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model) + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL + [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}" + src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" + tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" + if [[ -f "${src}" ]]; then + ${NCP} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + else + echo "WARNING: No mediator restarts available with warm_start=${EXP_WARM_START}" + fi + fi + fi + # Stage ice initial conditions to ROTDIR (warm start) if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART - [[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}" - src="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc" - tgt="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc" + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL + [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}" + src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc" + tgt="${COM_ICE_RESTART_PREV}/${PDY}.${cyc}0000.cice_model.res.nc" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -98,7 +141,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do YMD=${PDY} HH=${cyc} generate_com COM_WAVE_RESTART [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}" for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust - src="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}" + src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}" tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}" ${NCP} "${src}" "${tgt}" rc=$? @@ -108,6 +151,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do fi done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do + ############################################################### # Check for errors and exit if any of the above failed if [[ "${err}" -ne 0 ]]; then diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index a808cafd91..b1fa439052 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -432,7 +432,8 @@ def _gfs_or_gefs_forecast_args(parser): return parser def _gefs_args(parser): - parser.add_argument('--start', help=SUPPRESS, type=str, required=False, default='cold') + parser.add_argument('--start', help='restart mode: warm or cold', type=str, + choices=['warm', 'cold'], required=False, default='cold') parser.add_argument('--configdir', help=SUPPRESS, type=str, required=False, default=os.path.join(_top, 'parm/config/gefs')) parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False,