Add JEDI ATM lgetkf observer and solver jobs (NOAA-EMC#2833)

This PR adds JEDI ATM lgetkf observer and solver jobs to global-workflow. This is approach is akin GSI-based eobs and eupd. Splitting the single JEDI ATM lgetkf job into separate observer and solver jobs improves memory and computational efficiency. Resolves NOAA-EMC#2415
WalterKolczynski-NOAA · Aug 28, 2024 · 9ad7d3e · 9ad7d3e
1 parent 935ac64
commit 9ad7d3e
Show file tree

Hide file tree

Showing 31 changed files with 544 additions and 32 deletions.
diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -18,9 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
 
 skip_ci_on_hosts:
-  - hera
   - gaea
   - orion
   - hercules
-  - wcoss2
 
diff --git a/env/HERA.env b/env/HERA.env
@@ -72,6 +72,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -76,6 +76,16 @@ case ${step} in
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
  ;;
+ "atmensanlobs")
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+ ;;
+ "atmensanlsol")
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
+ ;;
  "atmensanlletkf")
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}

diff --git a/env/JET.env b/env/JET.env
@@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}

diff --git a/env/ORION.env b/env/ORION.env
@@ -68,6 +68,15 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}

diff --git a/env/S4.env b/env/S4.env
@@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -53,6 +53,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs"  ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -89,6 +99,33 @@ elif [[ "${step}" = "esnowrecen" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "marinebmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEBMAT="${APRUN}"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    export NTHREADS_OCNANALECEN=${NTHREADSmax}
+    export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
+elif [[ "${step}" = "marineanalletkf" ]]; then
+
+    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
+    export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS b/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS
@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+export WIPE_DATA="NO"
+export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}}
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlobs" -c "base atmensanl atmensanlobs"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASATMENSOBSSH:-${SCRgfs}/exglobal_atmens_analysis_obs.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0
diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL b/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL
@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+export WIPE_DATA="NO"
+export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}}
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlsol" -c "base atmensanl atmensanlsol"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASATMENSSOLSH:-${SCRgfs}/exglobal_atmens_analysis_sol.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0
diff --git a/jobs/rocoto/atmensanlobs.sh b/jobs/rocoto/atmensanlobs.sh
@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="atmensanlobs"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS"
+status=$?
+exit "${status}"
diff --git a/jobs/rocoto/atmensanlsol.sh b/jobs/rocoto/atmensanlsol.sh
@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="atmensanlsol"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL"
+status=$?
+exit "${status}"
diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2
@@ -15,17 +15,21 @@ enkf:
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log"
         {% endfor %}
 
-        {% if DO_JEDIATMENS %}
-        {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %}
-        {% else %}
-        {% set steps = ["eobs", "eupd"] %}
         {% if lobsdiag_forenkf %}
-        {% do steps.append("ediag") %}
+           {% if DO_JEDIATMENS %}
+              {% set steps = ["atmensanlinit", "atmensanlobs", "atmensanlsol", "atmensanlfv3inc", "atmensanlfinal"] %}
+           {% else %}
+              {% set steps = ["eobs", "ediag", "eupd"] %}
+           {% endif %}
         {% else %}
-        {% for mem in range(1, nmem_ens + 1) %}
-        {% do steps.append("eomg_mem{{ '%03d' % mem }}") %}
-        {% endfor %}
-        {% endif %}
+           {% if DO_JEDIATMENS %}
+              {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %}
+           {% else %}
+              {% set steps = ["eobs", "eupd"] %}
+              {% for mem in range(1, nmem_ens + 1) %}
+                 {% do steps.append("eomg_mem{{ '%03d' % mem }}") %}
+              {% endfor %}
+           {% endif %}
         {% endif %}
 
         {% for step in steps %}
@@ -49,10 +53,17 @@ enkf:
                            "oznstat.ensmean",
                            "radstat.ensmean"] %}
         {% else %}
+        {% if lobsdiag_forenkf %}
+        {% set da_files = ["atmens_observer.yaml",
+                           "atmens_solver.yaml",
+                           "atminc.ensmean.nc",
+                           "atmensstat"] %}
+        {% else %}
         {% set da_files = ["atmens.yaml",
                            "atminc.ensmean.nc",
                            "atmensstat"] %}
         {% endif %}
+        {% endif %}
         {% for file in da_files %}
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}"
         {% endfor %}

diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+########## config.atmensanlobs ##########
+# Pre Atm Ens Analysis specific
+
+echo "BEGIN: config.atmensanlobs"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" atmensanlobs
+
+export JCB_ALGO_YAML=@JCB_ALGO_YAML@
+
+echo "END: config.atmensanlobs"
diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+########## config.atmensanlsol ##########
+# Pre Atm Ens Analysis specific
+
+echo "BEGIN: config.atmensanlsol"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" atmensanlsol
+
+export JCB_ALGO_YAML=@JCB_ALGO_YAML@
+
+echo "END: config.atmensanlsol"
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -470,6 +470,7 @@ export ARCH_FCSTICFREQ=1     # Archive frequency in days for gdas and gfs foreca
 
 # The monitor jobs are not yet supported for JEDIATMVAR.
 if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
+   export DO_FIT2OBS="NO"     # Run fit to observations package    
    export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
    export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
    export DO_VMINMON="NO"     # GSI minimization monitoring

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -14,7 +14,7 @@ if (( $# != 1 )); then
     echo "stage_ic aerosol_init"
     echo "prep prepsnowobs prepatmiodaobs"
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
-    echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal"
+    echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal"
     echo "snowanl esnowrecen"
     echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst echgres"
@@ -286,7 +286,7 @@ case ${step} in
     ntasks=1
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
-    memory="3072M"
+    memory="4GB"
     ;;
 
   "atmanlvar")
@@ -1004,6 +1004,30 @@ case ${step} in
     memory="3072M"
     ;;
 
+  "atmensanlobs")
+    export layout_x=${layout_x_atmensanl}
+    export layout_y=${layout_y_atmensanl}
+
+    walltime="00:30:00"
+    ntasks=$(( layout_x * layout_y * 6 ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    memory="96GB"
+    export is_exclusive=True
+    ;;
+
+  "atmensanlsol")
+    export layout_x=${layout_x_atmensanl}
+    export layout_y=${layout_y_atmensanl}
+
+    walltime="00:30:00"
+    ntasks=$(( layout_x * layout_y * 6 ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    memory="96GB"
+    export is_exclusive=True
+    ;;
+
   "atmensanlletkf")
     export layout_x=${layout_x_atmensanl}
     export layout_y=${layout_y_atmensanl}