Add snow DA update and recentering for the EnKF forecasts

.gitignore

@@ -200,6 +200,7 @@ ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
+ush/bufr2ioda_insitu*
 
 # version files
 versions/build.ver

ci/Jenkinsfile

@@ -181,14 +181,14 @@ pipeline {
         }
 
         stage('4. Run Tests') {
-            failFast false 
+            failFast false
             matrix {
                 agent { label NodeName[machine].toLowerCase() }
                 axes {
                     axis {
                         name 'Case'
                         // TODO add dynamic list of cases from env vars (needs addtional plugins)
-                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
+                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96C48_hybatmaerosnowDA'
                     }
                 }
                 stages {
@@ -296,6 +296,6 @@ pipeline {
                     }
                 }
             }
-        }   
+        }
     }
 }

ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -6,12 +6,13 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATMA
   resdetatmos: 96
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
   idate: 2021122012
   edate: 2021122100
-  nens: 0
+  nens: 2
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

env/HERA.env

@@ -112,7 +112,17 @@ elif [[ "${step}" = "snowanl" ]]; then
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+
+elif [[ "${step}" = "esnowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
 
 elif [[ "${step}" = "ocnanalbmat" ]]; then
 

env/HERCULES.env

@@ -110,7 +110,17 @@ case ${step} in
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+ ;;
+ "esnowanl")
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
  ;;
  "ocnanalbmat")
 

env/JET.env

@@ -92,7 +92,17 @@ elif [[ "${step}" = "snowanl" ]]; then
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+
+elif [[ "${step}" = "esnowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
 
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 

env/ORION.env

@@ -100,7 +100,17 @@ elif [[ "${step}" = "snowanl" ]]; then
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl} --cpus-per-task=${NTHREADS_SNOWANL}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+
+elif [[ "${step}" = "esnowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
 
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 

env/S4.env

@@ -92,7 +92,17 @@ elif [[ "${step}" = "snowanl" ]]; then
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+
+elif [[ "${step}" = "esnowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
 
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 

env/WCOSS2.env

@@ -85,7 +85,17 @@ elif [[ "${step}" = "snowanl" ]]; then
     [[ ${NTHREADS_SNOWANL} -gt ${nth_max} ]] && export NTHREADS_SNOWANL=${nth_max}
     export APRUN_SNOWANL="${launcher} -n ${npe_snowanl}"
 
-    export APRUN_APPLY_INCR="${launcher} -n 6"
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
+
+elif [[ "${step}" = "esnowanl" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_esnowanl))
+
+    export NTHREADS_ESNOWANL=${nth_esnowanl:-${nth_max}}
+    [[ ${NTHREADS_ESNOWANL} -gt ${nth_max} ]] && export NTHREADS_ESNOWANL=${nth_max}
+    export APRUN_ESNOWANL="${launcher} -n ${npe_esnowanl} --cpus-per-task=${NTHREADS_ESNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n ${npe_apply_incr}"
 
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 

jobs/JGLOBAL_PREP_SNOW_OBS

@@ -41,4 +41,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0

jobs/JGLOBAL_SNOW_ANALYSIS

@@ -44,4 +44,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0

jobs/JGLOBAL_SNOW_ENSEMBLE_ANALYSIS

@@ -0,0 +1,59 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowanl" -c "base esnowanl"
+
+##############################################
+# Set variables used in the script
+##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDUMP="gdas"
+GDUMP_ENS="enkfgdas"
+export GDUMP
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMOUT_CONF:COM_CONF_TMPL
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \
+    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+
+mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
+
+for imem in $(seq 1 "${NMEM_ENS}"); do
+    memchar="mem$(printf %03i "${imem}")"
+    MEMDIR=${memchar} RUN=${GDUMP_ENS} YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+    mkdir -p "${COMOUT_SNOW_ANALYSIS}"
+done
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exglobal_snow_ensemble_analysis.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/esnowanl.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="esnowanl"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_SNOW_ENSEMBLE_ANALYSIS"
+status=$?
+exit "${status}"

parm/config/gfs/config.esnowanl

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+########## config.esnowanl ##########
+# configuration common to snow ensemble analysis tasks
+
+echo "BEGIN: config.esnowanl"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" esnowanl
+
+export JCB_BASE_YAML="${PARMgfs}/gdas/snow/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/snow/jcb-fv3jedi_land_ensrecenter.yaml.j2"
+
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export SNOW_ENS_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_ens_update.yaml.j2"
+export SNOW_OROG_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_orog.yaml.j2"
+export SNOW_ENS_FINALIZE_TMPL="${PARMgfs}/gdas/snow_finalize_ens_update.yaml.j2"
+
+# Name of the executable that applies increment to bkg and its namelist template
+export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
+export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/ens_apply_incr_nml.j2"
+
+export io_layout_x=@IO_LAYOUT_X@
+export io_layout_y=@IO_LAYOUT_Y@
+
+export JEDIEXE=${EXECgfs}/gdasapp_land_ensrecenter.x
+export FREGRID=${EXECgfs}/fregrid.x
+
+echo "END: config.esnowanl"

parm/config/gfs/config.resources

@@ -14,7 +14,7 @@ if (( $# != 1 )); then
     echo "prep prepsnowobs prepatmiodaobs"
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
     echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal"
-    echo "snowanl"
+    echo "snowanl esnowanl"
     echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst echgres"
     echo "upp atmos_products"
@@ -369,6 +369,37 @@ case ${step} in
     export npe_snowanl=$(( layout_x * layout_y * 6 ))
     export nth_snowanl=1
     export npe_node_snowanl=$(( npe_node_max / nth_snowanl ))
+    export npe_apply_incr=6
+    ;;
+
+  "esnowanl")
+    # below lines are for creating JEDI YAML
+    case ${CASE} in
+      "C768")
+        layout_x=6
+        layout_y=6
+        ;;
+      "C384")
+        layout_x=5
+        layout_y=5
+        ;;
+      "C192" | "C96" | "C48")
+        layout_x=1
+        layout_y=1
+        ;;
+      *)
+        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        exit 4
+    esac
+
+    export layout_x
+    export layout_y
+
+    export wtime_esnowanl="00:15:00"
+    export npe_esnowanl=$(( layout_x * layout_y * 6 ))
+    export nth_esnowanl=1
+    export npe_node_esnowanl=$(( npe_node_max / nth_esnowanl ))
+    export npe_apply_incr=6
     ;;
 
   "prepobsaero")