Refactor gempak jobs for new COM and style (NOAA-EMC#2374)

Updates the gempak jobs to fit the new COM structure while also
refactoring them some to improve the style. Despite these technical
changes, the overall structure is left unchanged for most scripts,
though some have been rewritten to make the needed changes easier. Some
of these scripts had already been updated some in the original COM
refactor and thus needed fewer updates.

Style updates includes converting all gempak scripts to bash, making
them shellcheck compliant, and removing trailing whitespace. Further
refactoring to improve maintainability will be needed in the future (see
NOAA-EMC#2341, NOAA-EMC#2342, NOAA-EMC#2343, NOAA-EMC#2348).

The GFS gif scripts were identical except the forecast hour, so they are
collapsed down into two: one for f000 and one for other forecast hours.

The gempak executables have short path limits. To get around this
without having the gempak module recompiled, target directories (mostly
relevant for the gempak meta jobs) are symlinked into the working
directory to drasticly reduce the path lengths.

Part of this update includes replacing existing MPMD calls with the new
standard `ush/run_mpmd.sh` script.

A new function, `wait_for_file()`, is introduced to standardize waiting
for a file to be available.

Gempak forecast hours are often hard-coded within scripts. In addition
to issues with maintainability, this causes problems for shorter
forecasts, such as we typically run for testing purposes. For now, we
simply check the values against the forecast length and reduce if
necessary. Future work (NOAA-EMC#2348) will be needed to remove these hard-coded
values with variables set in the config file (or just use update gempak
products to match standard output time variables).

One-degree gempak files have been updated to include `1p00` in the
filename.

Several gempak job dependencies are corrected.

Fake gempak data for external models is being staged on tier-1 machines
to allow testing.

**Output has not been verified.** Future PRs will likely be needed to
bring full functionality online.

Resolves NOAA-EMC#2158
Resolves NOAA-EMC#2152
Resolves NOAA-EMC#2151
Resolves NOAA-EMC#2249
Resolves NOAA-EMC#2247
Refs NOAA-EMC#2157
Refs NOAA-EMC#2348

.shellcheckrc

@@ -14,3 +14,6 @@ disable=SC1091
 
 # Disable -p -m only applies to deepest directory
 disable=SC2174
+
+# Disable warning of functions in test statements
+disable=SC2310

gempak/fix/datatype.tbl

@@ -102,10 +102,10 @@ LTNG         $OBS/ltng                 YYYYMMDDHH.ltng           CAT_MSC  SCAT_N
 !
 CLIMO        $GEMPAK/climo             climate_MM.mos            CAT_NIL  SCAT_NIL    1     -1     -1
 !
-GFS          $MODEL/gfs                gfs_YYYYMMDDHH            CAT_GRD  SCAT_FCT   -1     -1     -1
-F-GFS        $COMIN                    gfs_YYYYMMDDHHfFFF        CAT_GRD  SCAT_FCT   -1     -1     -1
-F-GFSP       $COMIN                    gfs_YYYYMMDDHHfFFF        CAT_GRD  SCAT_FCT   -1     -1     -1
-F-GFSHPC     $HPCGFS                   gfs_YYYYMMDDHHfFFF        CAT_GRD  SCAT_FCT   -1     -1     -1
+GFS          $MODEL/gfs                gfs_1p00_YYYYMMDDHH       CAT_GRD  SCAT_FCT   -1     -1     -1
+F-GFS        $COMIN                    gfs_1p00_YYYYMMDDHHfFFF   CAT_GRD  SCAT_FCT   -1     -1     -1
+F-GFSP       $COMIN                    gfs_1p00_YYYYMMDDHHfFFF   CAT_GRD  SCAT_FCT   -1     -1     -1
+F-GFSHPC     $HPCGFS                   gfs_1p00_YYYYMMDDHHfFFF   CAT_GRD  SCAT_FCT   -1     -1     -1
 GFSEXT       $MODEL/ens                gfs.YYYYMMDDHH            CAT_GRD  SCAT_FCT   -1     -1     -1
 GFS1         $MODEL/ens                gfs1.YYYYMMDDHH           CAT_GRD  SCAT_FCT   -1     -1     -1
 GFS2         $MODEL/ens                gfs2.YYYYMMDDHH           CAT_GRD  SCAT_FCT   -1     -1     -1
@@ -156,9 +156,9 @@ F-NAMP20     $COMIN                    nam20_YYYYMMDDHHfFFF      CAT_GRD  SCAT_F
 F-NAMP44     $COMIN                    nam44_YYYYMMDDHHfFFF      CAT_GRD  SCAT_FCT   -1     -1     -1
 F-THREATS    $COMIN                    ${NEST}_YYYYMMDDHHfFFF    CAT_GRD  SCAT_FCT   -1     -1     -1
 F-NAMHPC     $HPCNAM                   nam_YYYYMMDDHHfFFF        CAT_GRD  SCAT_FCT   -1     -1     -1
-GDAS         $MODEL/gdas               gdas_YYMMDDHH             CAT_GRD  SCAT_FCT   -1     -1     -1
-F-GDAS       $COMIN                    gdas_YYYYMMDDHHfFFF       CAT_GRD  SCAT_FCT   -1     -1     -1
-F-GFS        $COMIN                    gfs_YYYYMMDDHHfFFF        CAT_GRD  SCAT_FCT   -1     -1     -1
+GDAS         $MODEL/gdas               gdas_1p00_YYYYMMDDHH      CAT_GRD  SCAT_FCT   -1     -1     -1
+F-GDAS       $COMIN                    gdas_1p00_YYYYMMDDHHfFFF  CAT_GRD  SCAT_FCT   -1     -1     -1
+F-GFS        $COMIN                    gfs_1p00_YYYYMMDDHHfFFF   CAT_GRD  SCAT_FCT   -1     -1     -1
 F-HWRF       $COMIN                    hwrfp_YYYYMMDDHHfFFF_*    CAT_GRD  SCAT_FCT   -1     -1     -1
 F-HWRFN       $COMIN                   hwrfn_YYYYMMDDHHfFFF_*    CAT_GRD  SCAT_FCT   -1     -1     -1
 F-GHM        $COMIN                    ghmg_YYYYMMDDHHfFFF_*     CAT_GRD  SCAT_FCT   -1     -1     -1

gempak/ush/gdas_ecmwf_meta_ver.sh

@@ -1,75 +1,39 @@
-#!/bin/sh
-#
-# Metafile Script : gdas_ecmwf_meta_ver
+#! /usr/bin/env bash
 #
 # Creates a loop comparing the 6 hr gdas fcst to the pervious 7 days
 # of ecmwf fcsts
 #
-# Log :
-# J. Carr/HPC   3/2001   New metafile for verification of ecmwf.
-# J. Carr/HPC   5/2001   Added a mn variable for a/b side dbnet root variable.
-# M. Klein/HPC 11/2004   Changed verification grid from fnl to gdas
-# M. Klein/HPC  2/2005   Changed location of working directory to /ptmp
-# M. Klein/HPC 11/2006   Modify to run in production.
-#
-
-#cd $DATA
 
-set -xa
-
-if [ $cyc -ne "06" ] ; then
-    exit
-fi
+source "${HOMEgfs}/ush/preamble.sh"
 
-export pgm=gdplot2_nc;. prep_step; startmsg
+export pgm=gdplot2_nc;. prep_step
 
-cyc=12
+cyc2=12
 device="nc | ecmwfver.meta"
-PDY2=$(echo ${PDY} | cut -c3-)
 
 #
 # Copy in datatype table to define gdfile type
 #
-cp ${HOMEgfs}/gempak/fix/datatype.tbl datatype.tbl
+
+cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
 export err=$?
-if [[ $err -ne 0 ]] ; then
-   echo " File datatype.tbl does not exist."
-   exit $err
+if (( err != 0 )) ; then
+   echo "FATAL ERROR: File datatype.tbl does not exist."
+   exit "${err}"
 fi
 
-#
-# DEFINE YESTERDAY
-date1=$($NDATE -24 ${PDY}${cyc} | cut -c -8)
-sdate1=$(echo ${date1} | cut -c 3-)
-# DEFINE 2 DAYS AGO
-date2=$($NDATE -48 ${PDY}${cyc} | cut -c -8)
-sdate2=$(echo ${date2} | cut -c 3-)
-# DEFINE 3 DAYS AGO
-date3=$($NDATE -72 ${PDY}${cyc} | cut -c -8)
-sdate3=$(echo ${date3} | cut -c 3-)
-# DEFINE 4 DAYS AGO
-date4=$($NDATE -96 ${PDY}${cyc} | cut -c -8)
-sdate4=$(echo ${date4} | cut -c 3-)
-# DEFINE 5 DAYS AGO
-date5=$($NDATE -120 ${PDY}${cyc} | cut -c -8)
-sdate5=$(echo ${date5} | cut -c 3-)
-# DEFINE 6 DAYS AGO
-date6=$($NDATE -144 ${PDY}${cyc} | cut -c -8)
-sdate6=$(echo ${date6} | cut -c 3-)
-# DEFINE 7 DAYS AGO
-date7=$($NDATE -168 ${PDY}${cyc} | cut -c -8)
-sdate7=$(echo ${date7} | cut -c 3-)
-
-vergrid="F-GDAS | ${PDY2}/0600"
+export COMIN="gdas.${PDY}${cyc}"
+if [[ ! -L ${COMIN} ]]; then
+    ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
+fi
+vergrid="F-GDAS | ${PDY:2}/0600"
 fcsthr="0600f006"
 
 # GENERATING THE METAFILES.
 areas="SAM NAM"
-verdays="${date1} ${date2} ${date3} ${date4} ${date5} ${date6} ${date7}"
 
-for area in $areas
-    do
-    if [ $area == "NAM" ] ; then
+for area in ${areas}; do
+    if [[ "${area}" == "NAM" ]] ; then
         garea="5.1;-124.6;49.6;-11.9"
         proj="STR/90.0;-95.0;0.0"
         latlon="0"
@@ -80,37 +44,18 @@ for area in $areas
         latlon="1/10/1/2/10;10"
         run=" "
     fi
-    for verday in $verdays
-        do
-        verddate=$(echo ${verday} | cut -c 3-)
-        if [ ${verday} -eq ${date1} ] ; then
-            dgdattim=f024
-            sdatenum=$sdate1
-        elif [ ${verday} -eq ${date2} ] ; then
-            dgdattim=f048
-            sdatenum=$sdate2
-        elif [ ${verday} -eq ${date3} ] ; then
-            dgdattim=f072
-            sdatenum=$sdate3
-        elif [ ${verday} -eq ${date4} ] ; then
-            dgdattim=f096
-            sdatenum=$sdate4
-        elif [ ${verday} -eq ${date5} ] ; then
-            dgdattim=f120
-            sdatenum=$sdate5
-        elif [ ${verday} -eq ${date6} ] ; then
-            dgdattim=f144
-            sdatenum=$sdate6
-        elif [ ${verday} -eq ${date7} ] ; then
-            dgdattim=f168
-            sdatenum=$sdate7
+    for (( fhr=24; fhr<=168; fhr+=24 )); do
+        dgdattim=$(printf "f%03d" "${fhr}")
+        sdatenum=$(date --utc +%y%m%d -d "${PDY} ${cyc2} - ${fhr} hours")
+
+        if [[ ! -L "ecmwf.20${sdatenum}" ]]; then
+            ln -sf "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
         fi
-        # JY grid="$COMROOT/nawips/${envir}/ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
-        grid="${COMINecmwf}.20${sdatenum}/gempak/ecmwf_glob_20${sdatenum}12"
+        gdfile="ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
 
-# 500 MB HEIGHT METAFILE
+        # 500 MB HEIGHT METAFILE
 
-$GEMEXE/gdplot2_nc << EOFplt
+        "${GEMEXE}/gdplot2_nc" << EOFplt
 \$MAPFIL = mepowo.gsf
 PROJ     = ${proj}
 GAREA    = ${garea}
@@ -134,7 +79,7 @@ line     = 6/1/3
 title    = 6/-2/~ GDAS 500 MB HGT (6-HR FCST)|~${area} 500 HGT DF
 r
 
-gdfile   = ${grid}
+gdfile   = ${gdfile}
 gdattim  = ${dgdattim}
 line     = 5/1/3
 contur   = 4
@@ -157,15 +102,15 @@ clear    = yes
 latlon   = ${latlon}
 r
 
-gdfile   = ${grid}
+gdfile   = ${gdfile}
 gdattim  = ${dgdattim}
 line     = 5/1/3
 contur   = 4
 title    = 5/-1/~ ECMWF PMSL
 clear    = no
 r
 
-PROJ     = 
+PROJ     =
 GAREA    = bwus
 gdfile   = ${vergrid}
 gdattim  = ${fcsthr}
@@ -181,7 +126,7 @@ clear    = yes
 latlon   = ${latlon}
 ${run}
 
-gdfile   = ${grid}
+gdfile   = ${gdfile}
 gdattim  = ${dgdattim}
 line     = 5/1/3
 contur   = 4
@@ -195,28 +140,28 @@ EOFplt
     done
 done
 
-export err=$?;err_chk
+export err=$?
+
 #####################################################
 # GEMPAK DOES NOT ALWAYS HAVE A NON ZERO RETURN CODE
 # WHEN IT CAN NOT PRODUCE THE DESIRED GRID.  CHECK
 # FOR THIS CASE HERE.
 #####################################################
-ls -l ecmwfver.meta
-export err=$?;export pgm="GEMPAK CHECK FILE";err_chk
-
-if [ $SENDCOM = "YES" ] ; then
-    mkdir -p -m 775 ${COMOUTecmwf}.${PDY}/meta
-    mv ecmwfver.meta ${COMOUTecmwf}.${PDY}/meta/ecmwfver_${PDY}_${cyc}
-    export err=$?
-    if [[ $err -ne 0 ]] ; then
-       echo " File ecmwfver.meta does not exist."
-       exit $err
-    fi
+if (( err != 0 )) || [[ ! -s ecmwfver.meta ]]; then
+    echo "FATAL ERROR: Failed to create ecmwf meta file"
+    exit "${err}"
+fi
 
-    if [ $SENDDBN = "YES" ] ; then
-        ${DBNROOT}/bin/dbn_alert MODEL ECMWFVER_HPCMETAFILE $job \
-        ${COMOUTecmwf}.${PDY}/meta/ecmwfver_${PDY}_${cyc}
-    fi
+mv ecmwfver.meta "${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
+export err=$?
+if (( err != 0 )) ; then
+    echo "FATAL ERROR: Failed to move meta file to ${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
+    exit "${err}"
+fi
+
+if [[ "${SENDDBN}" == "YES" ]] ; then
+    "${DBNROOT}/bin/dbn_alert" MODEL ECMWFVER_HPCMETAFILE "${job}" \
+        "${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
 fi
 
 exit