diff --git a/jobs/JGEFS_ATMOS_ACC b/jobs/JGEFS_ATMOS_ACC new file mode 100755 index 0000000000..9c133fe80f --- /dev/null +++ b/jobs/JGEFS_ATMOS_ACC @@ -0,0 +1,28 @@ +#! /usr/bin/env bash +# +# +##################################################################### +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "repair_replay" -c "base repair_replay" + +#################################### +# Specify Execution Areas +#################################### +# +# Construct COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \ + COMOUT_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL + +export COMIN_03=${HOMEgfs}/anl + +######################################################## +# Execute the acc script. +"${SCRgfs}/exgefs_atmos_repair_replay.sh" +export err=$?; err_chk +############################## +# Remove the Temporary working directory +############################## +if [[ "${KEEPDATA}" != "YES" ]] ; then + rm -fr "${DATA}" +fi diff --git a/jobs/rocoto/repair_replay.sh b/jobs/rocoto/repair_replay.sh new file mode 100755 index 0000000000..7175a65adf --- /dev/null +++ b/jobs/rocoto/repair_replay.sh @@ -0,0 +1,22 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +echo +echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="repair_replay" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGEFS_ATMOS_ACC" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + + +exit 0 diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index fad9e3421a..3606a8719a 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -133,6 +133,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk +export DO_REPAIR_REPLAY="@DO_REPAIR_REPLAY@" # Option to repair ave/acc/min/max variabes for f03 and f06 when one uses reply initial condition export AERO_FCST_RUN="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_RUN="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_RUN="" # When to include wave suite: gdas, gfs, or both diff --git a/parm/config/gefs/config.repair_replay b/parm/config/gefs/config.repair_replay new file mode 100644 index 0000000000..87b41d9fe7 --- /dev/null +++ b/parm/config/gefs/config.repair_replay @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +########## config.repair_replay ######### +# repair_replay specific + +echo "BEGIN: config.repair_replay" + + +export varlist_FLXacc03="${PARMgfs}/product/gefs_parmlist_FLXacc03.parm" # Parameter table for f03 acc/ave/min/max variables (PSL data) +export varlist_masteracc06="${PARMgfs}/product/gefs_parmlist_masteracc06.parm" # Parameter table for f06 acc/ave/min/max variables +export varlist_masteracc03="${PARMgfs}/product/gefs_parmlist_masteracc03.parm" # Parameter table for f06 acc/ave/min/max variables + +. "${EXPDIR}/config.resources" repair_replay + +echo "END: config.repair_replay" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 297bc08c05..97fdb3ffb9 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -77,6 +77,14 @@ case ${step} in export memory="1GB" ;; + "repair_replay") + export walltime="00:10:00" + export ntasks=1 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export memory="2GB" + ;; + "fcst" | "efcs") export is_exclusive=True @@ -194,6 +202,7 @@ case ${step} in unset NTASKS_TOT ;; + "atmos_products") export walltime="00:15:00" export ntasks=24 diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index e4666d1aba..b420a62e7b 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -9,6 +9,7 @@ base: DO_AWIPS: "NO" KEEPDATA: "NO" DO_EXTRACTVARS: "NO" + DO_REPAIR_REPLAY: "NO" FHMAX_GFS: 120 FHMAX_HF_GFS: 0 FCST_BREAKPOINTS: "48" diff --git a/parm/product/gefs_parmlist_FLXacc03.parm b/parm/product/gefs_parmlist_FLXacc03.parm new file mode 100644 index 0000000000..6326ea97da --- /dev/null +++ b/parm/product/gefs_parmlist_FLXacc03.parm @@ -0,0 +1,35 @@ +:WATR:surface:0-3 hour acc fcst: +:SNOWC:surface:0-3 hour ave fcst: +:SNOHF:surface:0-3 hour ave fcst: +:DLWRF:surface:0-3 hour ave fcst: +:ULWRF:surface:0-3 hour ave fcst: +:DSWRF:surface:0-3 hour ave fcst: +:USWRF:surface:0-3 hour ave fcst: +:USWRF:top of atmosphere:0-3 hour ave fcst: +:ULWRF:top of atmosphere:0-3 hour ave fcst: +:UFLX:surface:0-3 hour ave fcst: +:VFLX:surface:0-3 hour ave fcst: +:SHTFL:surface:0-3 hour ave fcst: +:LHTFL:surface:0-3 hour ave fcst: +:PRATE:surface:0-3 hour ave fcst: +:CPRAT:surface:0-3 hour ave fcst: +:ALBDO:surface:0-3 hour ave fcst: +:TCDC:entire atmosphere (considered as a single layer):0-3 hour ave fcst: +:TCDC:boundary layer cloud layer:0-3 hour ave fcst: +:GFLUX:surface:0-3 hour ave fcst: +:U-GWD:surface:0-3 hour ave fcst: +:V-GWD:surface:0-3 hour ave fcst: +:TMP:middle cloud top level:0-3 hour ave fcst: +:TMP:low cloud top level:0-3 hour ave fcst: +:TMP:high cloud top level:0-3 hour ave fcst: +:PRES:high cloud top level:0-3 hour ave fcst: +:PRES:middle cloud top level:0-3 hour ave fcst: +:PRES:low cloud top level:0-3 hour ave fcst: +:PRES:high cloud bottom level:0-3 hour ave fcst: +:PRES:middle cloud bottom level:0-3 hour ave fcst: +:PRES:low cloud bottom level:0-3 hour ave fcst: +:CWORK:entire atmosphere (considered as a single layer):0-3 hour ave fcst: +:DUVB:surface:0-3 hour ave fcst: +:CDUVB:surface:0-3 hour ave fcst: +:TMAX:2 m above ground:0-3 hour max fcst: +:TMIN:2 m above ground:0-3 hour min fcst: diff --git a/parm/product/gefs_parmlist_masteracc03.parm b/parm/product/gefs_parmlist_masteracc03.parm new file mode 100644 index 0000000000..19abd35066 --- /dev/null +++ b/parm/product/gefs_parmlist_masteracc03.parm @@ -0,0 +1,42 @@ +:TSNOWP +:DLWRF:surface +:ULWRF:surface +:DSWRF:surface +:USWRF:surface +:USWRF:top +:ULWRF:top +:UFLX +:VFLX +:SHTFL +:LHTFL +:PRATE +:CPRAT +:ALBDO +:GFLUX +:U-GWD +:V-GWD +:TCDC:entire +:TCDC:boundary +:HCDC:high cloud layer:182-185 min +:LCDC:low cloud layer:182-185 min +:MCDC:middle cloud layer:182-185 min +:APCP +:ACPCP +:NCPCP +:TMIN +:TMAX +:DUVB +:CDUVB +:CWORK +:SNOHF +:SNOWC +:WATR +:TMP:middle cloud top +:TMP:low cloud top +:TMP:high cloud top +:PRES:high cloud top +:PRES:middle cloud top +:PRES:low cloud top +:PRES:high cloud bottom +:PRES:middle cloud bottom +:PRES:low cloud bottom diff --git a/parm/product/gefs_parmlist_masteracc06.parm b/parm/product/gefs_parmlist_masteracc06.parm new file mode 100644 index 0000000000..50847e2b25 --- /dev/null +++ b/parm/product/gefs_parmlist_masteracc06.parm @@ -0,0 +1,41 @@ +:WATR +:SNOWC +:SNOHF +:DLWRF:surface +:ULWRF:surface +:DSWRF:surface +:USWRF:surface +:USWRF:top +:ULWRF:top +:UFLX:surface +:VFLX:surface +:SHTFL:surface +:LHTFL:surface +:PRATE:surface:0 +:CPRAT:surface:0 +:ALBDO:surface +:TCDC:entire +:TCDC:boundary +:HCDC:high cloud layer:0-6 +:MCDC:middle cloud layer:0-6 +:LCDC:low cloud layer:0-6 +:GFLUX:surface +:U-GWD:surface +:V-GWD:surface +:APCP:surface +:ACPCP:surface +:NCPCP:surface +:TMP:middle cloud top +:TMP:low cloud top +:TMP:high cloud top +:PRES:high cloud top +:PRES:middle cloud top +:PRES:low cloud top +:PRES:high cloud bottom +:PRES:middle cloud bottom +:PRES:low cloud bottom +:CWORK +:DUVB +:CDUVB +:TMAX +:TMIN diff --git a/scripts/exgefs_atmos_repair_replay.sh b/scripts/exgefs_atmos_repair_replay.sh new file mode 100755 index 0000000000..5d807bf4c1 --- /dev/null +++ b/scripts/exgefs_atmos_repair_replay.sh @@ -0,0 +1,130 @@ +#! /usr/bin/env bash +# shellcheck disable=SC2312 + + +source "${USHgfs}/preamble.sh" + +cd "${DATA}" || exit 1 + +#COMIN_03 and {COMIN_ATMOS_MASTER} are directory containing the files that we want to extract +#Extract c00 files for f003 from PSL reanalysis data + + +fnh=03 +echo "extracting f${fnh}" +oufile=${DATA}/gefs.t00z.master.grb2f0${fnh} + +infile=${COMIN_03}/GFSPRS.GrbF03 +if [[ -f "${infile}" ]]; then #check if input file exists before extraction + ${WGRIB2} "${infile}" | grep "TSNOWP" | ${WGRIB2} -i "${infile}" -grib tmp + export err=$?; err_chk + + ${WGRIB2} tmp -for "2:2" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":APCP:surface" | ${WGRIB2} -i "${infile}" -grib tmp + export err=$?; err_chk + + ${WGRIB2} tmp -for "1:1" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":ACPCP:surface" | ${WGRIB2} -i "${infile}" -grib tmp + export err=$?; err_chk + + ${WGRIB2} tmp -for "1:1" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":NCPCP:surface" | ${WGRIB2} -i "${infile}" -grib tmp + export err=$?; err_chk + + ${WGRIB2} tmp -for "1:1" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":HCDC:high cloud layer:0" | ${WGRIB2} -i "${infile}" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":MCDC:middle cloud layer:0" | ${WGRIB2} -i "${infile}" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + ${WGRIB2} "${infile}" | grep ":LCDC:low cloud layer:0" | ${WGRIB2} -i "${infile}" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + + rm tmp +else + echo "FATAL ERROR: ${infile} does not exist" + export err=1; err_chk +fi + +varlist=${varlist_FLXacc03} # Parameter table for f03 acc/ave/min/max variables (PSL data) +infile=${COMIN_03}/GFSFLX.GrbF03 +if [[ -f "${infile}" ]]; then #check if input file exists before extraction + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile}">/dev/null + export err=$?; err_chk +else + echo "FATAL ERROR: ${infile} does not exist" + export err=1; err_chk +fi +# done + + +#Extract individual member files for f006 master data +fnh=006 + +echo "extracting f${fnh}" +infile=${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f${fnh} +oufile=${DATA}/gefs.t00z.master.grb2f${fnh} + +varlist=${varlist_masteracc06} # Parameter table for f06 acc/ave/min/max variables + +if [[ -f "${infile}" ]]; then #check if input file exists before extraction + + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile}" + export err=$?; err_chk + ${WGRIB2} "${infile}" | grep "TSNOWP" | ${WGRIB2} -i "${infile}" -grib tmp + export err=$?; err_chk + ${WGRIB2} tmp -for "2:2" -append -grib "${oufile}">/dev/null + export err=$?; err_chk + ${WGRIB2} tmp -for "1:1" -grib TSNOWP1.dat + export err=$?; err_chk + +else + echo "FATAL ERROR: ${infile} does not exist" + export err=1; err_chk +fi + +export sorc_name=gefs_6h_ave_1mem.x + +"${EXECgfs}/${sorc_name}" > sorc_name.exe.out +cat sorc_name.exe.out + +#output f06 +infile=${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f006 + +${WGRIB2} "${infile}" -match_inv | grep -v -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -grib out1.grb2 +${WGRIB2} out1.grb2 -not "TSNOWP" -grib out2.grb2 + +cat out2.grb2 TSNOWP1.dat gefs.t00z.pgrb2af006 > out3.grb2 +mv "${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f006" "${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f006_org" +mv out3.grb2 "${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f006" + +rm -fr out1.grb2 out2.grb2 TSNOWP*.dat + +#output f03 +varlist=${varlist_masteracc03} # Parameter table for f03 acc/ave/min/max variables +infile=${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f003 +# +${WGRIB2} "${infile}" | grep "TSNOWP" | ${WGRIB2} -i "${infile}" -grib TSNOWP2.dat +${WGRIB2} TSNOWP2.dat -for "1:1" -grib out1.grb2 >/dev/null +${WGRIB2} out1.grb2 -set_ftime "0-3 hour acc fcst" -grib TSNOWP1.dat + +${WGRIB2} "${infile}" -match_inv | grep -v -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -grib out2.grb2 +${WGRIB2} out2.grb2 -set_ftime "3 hour fcst" -grib out3.grb2 + +cat out3.grb2 TSNOWP1.dat gefs.t00z.pgrb2af003 > gefs.t00z.master.grb2f003 + +${NCP} "${COMIN_ATMOS_MASTER}/gefs.t00z.master.grb2f003" "${COMOUT_ATMOS_MASTER}/gefs.t00z.master.grb2f003_org" +${NCP} gefs.t00z.master.grb2f003 "${COMOUT_ATMOS_MASTER}/gefs.t00z.master.grb2f003" + +rm -fr out1.grb2 out2.grb2 out3.grb2 + +exit diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 279bbf2097..b9b965317e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 279bbf2097d87321294436d17bf5b73c4c07ab4a +Subproject commit b9b965317e74243d8404a609b5799186850ef03c diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index ae30e7a645..ff6f151298 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -317,7 +317,7 @@ cd "${HOMEgfs}/exec" || exit 1 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \ syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \ mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \ - ensadd.x ensppf.x ensstat.x wave_stat.x + ensadd.x ensppf.x ensstat.x wave_stat.x gefs_6h_ave_1mem.x do [[ -s "${utilexe}" ]] && rm -f "${utilexe}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" . diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 8c1f69735e..62dd91f26b 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -71,6 +71,7 @@ def __init__(self, conf: Configuration) -> None: self.do_goes = _base.get('DO_GOES', False) self.do_mos = _base.get('DO_MOS', False) self.do_extractvars = _base.get('DO_EXTRACTVARS', False) + self.do_repair_replay = _base.get('DO_REPAIR_REPLAY', False) self.do_hpssarch = _base.get('HPSSARCH', False) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c1e001c171..f20e871b9d 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -14,11 +14,14 @@ def _get_app_configs(self): """ Returns the config_files that are involved in gefs """ - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch'] + configs = ['stage_ic', 'fcst', 'repair_replay', 'atmos_products', 'arch'] if self.nens > 0: configs += ['efcs', 'atmos_ensstat'] + if self.do_repair_replay: + configs += ['repair_replay'] + if self.do_wave: configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] if self.do_wave_bnd: @@ -59,6 +62,9 @@ def get_task_names(self): if self.nens > 0: tasks += ['efcs'] + if self.do_repair_replay: + tasks += ['repair_replay'] + tasks += ['atmos_prod'] if self.nens > 0: diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f0f73d1173..e90f9b3072 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -169,6 +169,50 @@ def fcst(self): return task + def atmos_prod(self): + return self._atmosoceaniceprod('atmos') + + def repair_replay(self): + + deps = [] + history_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_MASTER_TMPL"], {'MEMDIR': 'mem#member#'}) + data = f'{history_path}/{self.run}.t@Hz.master.grb2f006' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep=deps) + + repair_replay_envars = self.envars.copy() + repair_replayenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#' + } + + for key, value in repair_replayenvar_dict.items(): + repair_replay_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('repair_replay') + task_name = f'repair_replay_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': repair_replay_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/repair_replay.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'repair_replay', + 'task_dict': task_dict, + 'var_dict': member_var_dict, + } + + task = rocoto.create_task(member_metatask_dict) + + return task + def efcs(self): dependencies = [] dep_dict = {'type': 'task', 'name': f'stage_ic'} @@ -239,9 +283,6 @@ def efcs(self): # return task - def atmos_prod(self): - return self._atmosoceaniceprod('atmos') - def ocean_prod(self): return self._atmosoceaniceprod('ocean') @@ -286,7 +327,10 @@ def _atmosoceaniceprod(self, component: str): else: dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + if self.app_config.do_repair_replay: + dep_dict = {'type': 'task', 'name': 'repair_replay_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#',