Add snow DA update and recentering for the EnKF forecasts (#2690)

This PR adds the capability to update the ensemble of snow states by recentering the ensemble mean to the deterministic snow analysis and applying increments as appropriate. Resolves #2585 --------- Co-authored-by: David Huber <[email protected]> Co-authored-by: Rahul Mahajan <[email protected]> Co-authored-by: Guillaume Vernieres <[email protected]> Co-authored-by: AntonMFernando-NOAA <[email protected]> Co-authored-by: Anil Kumar <[email protected]> Co-authored-by: TerrenceMcGuinness-NOAA <[email protected]>
NOAA-EMC · Aug 23, 2024 · 2ce2116 · 2ce2116
1 parent 1b18f2f
commit 2ce2116
Show file tree

Hide file tree

Showing 29 changed files with 911 additions and 42 deletions.
diff --git a/.gitignore b/.gitignore
@@ -200,6 +200,7 @@ ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
+ush/bufr2ioda_insitu*
 
 # version files
 versions/build.ver

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -267,7 +267,7 @@ pipeline {
                 }
             }
         }
-            
+
 
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
@@ -297,6 +297,6 @@ pipeline {
                     }
                 }
             }
-        }   
+        }
     }
 }
diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml
@@ -6,12 +6,13 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATMA
   resdetatmos: 96
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122012
   edate: 2021122100
-  nens: 0
+  nens: 2
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -22,4 +22,5 @@ skip_ci_on_hosts:
   - gaea
   - orion
   - hercules
+  - wcoss2
 
diff --git a/env/HERA.env b/env/HERA.env
@@ -80,7 +80,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -106,6 +106,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -105,6 +105,14 @@ case ${step} in
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;
+ "esnowrecen")
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+ ;;
+
  "marinebmat")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

diff --git a/env/JET.env b/env/JET.env
@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/ORION.env b/env/ORION.env
@@ -97,6 +97,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/S4.env b/env/S4.env
@@ -68,7 +68,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/jobs/JGDAS_ENKF_SNOW_RECENTER b/jobs/JGDAS_ENKF_SNOW_RECENTER
@@ -0,0 +1,59 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen"
+
+##############################################
+# Set variables used in the script
+##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDUMP="gdas"
+export GDUMP
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMOUT_CONF:COM_CONF_TMPL
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \
+    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+
+mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
+
+for imem in $(seq 1 "${NMEM_ENS}"); do
+    memchar="mem$(printf %03i "${imem}")"
+    MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+        COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+    mkdir -p "${COMOUT_SNOW_ANALYSIS}"
+done
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exgdas_enkf_snow_recenter.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0
diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS
@@ -41,4 +41,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0
diff --git a/jobs/JGLOBAL_SNOW_ANALYSIS b/jobs/JGLOBAL_SNOW_ANALYSIS
@@ -44,4 +44,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0
diff --git a/jobs/rocoto/esnowrecen.sh b/jobs/rocoto/esnowrecen.sh
@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="esnowrecen"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_ENKF_SNOW_RECENTER"
+status=$?
+exit "${status}"
diff --git a/parm/config/gfs/config.esnowrecen b/parm/config/gfs/config.esnowrecen
@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+########## config.esnowrecen ##########
+# configuration common to snow ensemble analysis tasks
+
+echo "BEGIN: config.esnowrecen"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" esnowrecen
+
+export JCB_BASE_YAML="${PARMgfs}/gdas/snow/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/snow/jcb-fv3jedi_land_ensrecenter.yaml.j2"
+
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export SNOW_ENS_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_ens_update.yaml.j2"
+export SNOW_OROG_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_orog.yaml.j2"
+export SNOW_ENS_FINALIZE_TMPL="${PARMgfs}/gdas/snow_finalize_ens_update.yaml.j2"
+
+# Name of the executable that applies increment to bkg and its namelist template
+export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
+export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/ens_apply_incr_nml.j2"
+
+export io_layout_x=@IO_LAYOUT_X@
+export io_layout_y=@IO_LAYOUT_Y@
+
+export JEDIEXE=${EXECgfs}/gdasapp_land_ensrecenter.x
+export FREGRID=${EXECgfs}/fregrid.x
+
+echo "END: config.esnowrecen"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -15,7 +15,7 @@ if (( $# != 1 )); then
     echo "prep prepsnowobs prepatmiodaobs"
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
     echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal"
-    echo "snowanl"
+    echo "snowanl esnowrecen"
     echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst echgres"
     echo "upp atmos_products"
@@ -348,6 +348,35 @@ case ${step} in
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
+  "esnowrecen")
+    # below lines are for creating JEDI YAML
+    case ${CASE} in
+      "C768")
+        layout_x=6
+        layout_y=6
+        ;;
+      "C384")
+        layout_x=5
+        layout_y=5
+        ;;
+      "C192" | "C96" | "C48")
+        layout_x=1
+        layout_y=1
+        ;;
+      *)
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
+        exit 4
+    esac
+
+    export layout_x
+    export layout_y
+
+    walltime="00:15:00"
+    ntasks=$(( layout_x * layout_y * 6 ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    ;;
+
   "prepobsaero")
     walltime="00:30:00"
     ntasks=1

diff --git a/parm/gdas/snow_finalize_ens_update.yaml.j2 b/parm/gdas/snow_finalize_ens_update.yaml.j2
@@ -0,0 +1,43 @@
+copy:
+######################################
+# copy analyses to directories
+######################################
+{% for mem in range(1, NMEM_ENS + 1) %}
+    # define variables
+    # Declare a dict of search and replace terms to run on each template
+    {% set tmpl_dict = {'${ROTDIR}':ROTDIR,
+                        '${RUN}':RUN,
+                        '${YMD}':current_cycle | to_YMD ,
+                        '${HH}':current_cycle | strftime("%H"),
+                        '${MEMDIR}':"mem" + '%03d' % mem} %}
+
+    {% for tile in range(1, ntiles+1) %}
+- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
+    {% endfor %}
+    {% if DOIAU == True %}
+        # if using IAU, also need analyses copied at the beginning of the window
+        {% for tile in range(1, ntiles+1) %}
+- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
+        {% endfor %}
+    {% endif %}
+{% endfor %}
+######################################
+# copy ensemble mean increment to COM
+######################################
+# define variables
+# Declare a dict of search and replace terms to run on each template
+{% set tmpl_dict = {'${ROTDIR}':ROTDIR,
+                    '${RUN}':RUN,
+                    '${YMD}':current_cycle | to_YMD ,
+                    '${HH}':current_cycle | strftime("%H"),
+                    '${MEMDIR}':"ensstat"} %}
+
+{% for tile in range(1, ntiles+1) %}
+- ["{{ DATA }}/inc/ensmean/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
+{% endfor %}
+{% if DOIAU == True %}
+    # if using IAU, also need increment copied at the beginning of the window
+    {% for tile in range(1, ntiles+1) %}
+- ["{{ DATA }}/inc/ensmean/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"]
+    {% endfor %}
+{% endif %}