Separate offline UPP from interpolated grib2 product generation (NOAA…

…-EMC#2110)

This PR:
- Replaces the use of  `JGLOBAL_ATMOS_POST` with `JGLOBAL_ATMOS_UPP`
and `JGLOBAL_ATMOS_PRODUCTS`.
- Introduces `JGLOBAL_ATMOS_PRODUCTS` that is responsible for creating
grib2 products from the master file produced by UPP (inline from the model,
or offline by running `JGLOBAL_ATMOS_UPP`).  Rocoto job `atmos_products.sh`,
ex-script `exglobal_atmos_products.sh` are also added along with a
`config.atmos_products`. Updates are made to the relevant `env` files as well as
`config.resources` to reflect the addition of this job.
- reduces the number of cores needed appropriately for forecast products when
there is no offline UPP necessary.
- renames older `fv3gfs_dwn_nems.sh` to `interp_atmos_master.sh` and
`inter_flux.sh` to `interp_atmos_sflux.sh`
- removes `fv3gfs_downstream_nems.sh` as it is absorbed into
`exglobal_atmos_products.sh`
- removes no longer used scripts for `gfs_post.sh`, `gfs_transfer.sh` (DBN alerts
are absorbed into `exglobal_atmos_products.sh`), `post.sh`,
`JGLOBAL_ATMOS_POST`, `ex${RUN}_atmos_post.sh` scripts.
- Rocoto workflow related scripts are updated to add `atmos_products` as a job.
When `WRITE_DOPOST` is `.false.`, `upp.sh` job to run the UPP offline is also
added.  Dependencies have been appropriately applied.
- removes `PGB1F` option that creates 1-degree grib1 products that are no longer
needed in GFSv17.

The hacks from (now deleted) `post.sh` had to be reinstated in `upp.sh` to load the
modules from UPP instead of `load_fv3gfs_modules.sh`.

env/HERA.env

@@ -206,18 +206,9 @@ elif [[ "${step}" = "upp" ]]; then
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
 
-elif [[ "${step}" = "post" ]]; then
-
-    nth_max=$((npe_node_max / npe_node_post))
-
-    export NTHREADS_NP=${nth_np:-1}
-    [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher} -n ${npe_post} --cpus-per-task=${NTHREADS_NP}"
+elif [[ "${step}" = "atmos_products" ]]; then
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
-    export NTHREADS_DWN=${nth_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN="${launcher} -n ${npe_dwn} --cpus-per-task=${NTHREADS_DWN}"
 
 elif [[ "${step}" = "ecen" ]]; then
 
@@ -302,4 +293,3 @@ elif [[ "${step}" = "fit2obs" ]]; then
     export MPIRUN="${launcher} -n ${npe_fit2obs} --cpus-per-task=${NTHREADS_FIT2OBS}"
 
 fi
-

env/JET.env

@@ -193,18 +193,9 @@ elif [[ "${step}" = "upp" ]]; then
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp}"
 
-elif [[ "${step}" = "post" ]]; then
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_post))
-
-    export NTHREADS_NP=${nth_np:-1}
-    [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher} -n ${npe_post}"
-
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Jet
-    export NTHREADS_DWN=${nth_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN="${launcher} -n ${npe_dwn}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation
 
 elif [[ "${step}" = "ecen" ]]; then
 

env/ORION.env

@@ -206,18 +206,9 @@ elif [[ "${step}" = "upp" ]]; then
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
 
-elif [[ "${step}" = "post" ]]; then
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_post))
-
-    export NTHREADS_NP=${nth_np:-1}
-    [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher} -n ${npe_post} --cpus-per-task=${NTHREADS_NP}"
-
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Orion
-    export NTHREADS_DWN=${nth_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN="${launcher} -n ${npe_dwn} --cpus-per-task=${NTHREADS_DWN}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation
 
 elif [[ "${step}" = "ecen" ]]; then
 

env/S4.env

@@ -179,18 +179,9 @@ elif [[ "${step}" = "upp" ]]; then
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp}"
 
-elif [[ "${step}" = "post" ]]; then
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_post))
-
-    export NTHREADS_NP=${nth_np:-1}
-    [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher} -n ${npe_post}"
-
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on S4
-    export NTHREADS_DWN=${nth_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN="${launcher} -n ${npe_dwn}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation
 
 elif [[ "${step}" = "ecen" ]]; then
 

env/WCOSS2.env

@@ -191,18 +191,9 @@ elif [[ "${step}" = "upp" ]]; then
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp} -ppn ${npe_node_upp} --cpu-bind depth --depth ${NTHREADS_UPP}"
 
-elif [[ "${step}" = "post" ]]; then
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    nth_max=$((npe_node_max / npe_node_post))
-
-    export NTHREADS_NP=${nth_np:-1}
-    [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP="${launcher} -n ${npe_np:-${npe_post}} -ppn ${npe_node_post} --cpu-bind depth --depth ${NTHREADS_NP}"
-
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on WCOSS2
-    export NTHREADS_DWN=${nth_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN="${launcher} -np ${npe_dwn} ${mpmd_opt}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation
 
 elif [[ "${step}" = "ecen" ]]; then
 

jobs/JGLOBAL_ATMOS_PRODUCTS

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmos_products" -c "base atmos_products"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Construct COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_HISTORY COM_ATMOS_MASTER
+
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COM_ATMOS_GRIB_${grid}"
+  GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+done
+
+# Variables used in this job
+export PREFIX="${RUN}.t${cyc}z."
+
+###############################################################
+# Run exglobal script
+"${HOMEgfs}/scripts/exglobal_atmos_products.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA:-NO}" = "NO" ]] && rm -rf "${DATA}"
+
+
+exit 0

jobs/rocoto/atmos_products.sh

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+## atmosphere products driver script
+## FHRLST : forecast hour list to post-process (e.g. -f001, f000, f000_f001_f002, ...)
+###############################################################
+
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+if (( status != 0 )); then exit "${status}"; fi
+
+export job="atmos_products"
+export jobid="${job}.$$"
+
+###############################################################
+# shellcheck disable=SC2153,SC2001
+IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+
+#---------------------------------------------------------------
+# Execute the JJOB
+for fhr in "${fhrs[@]}"; do
+    export FORECAST_HOUR=$(( 10#${fhr} ))
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_PRODUCTS"
+    status=$?
+    if (( status != 0 )); then exit "${status}"; fi
+done
+
+exit 0