Merge remote-tracking branch 'origin/develop' into fix/restartb_archi…

…ving * origin/develop: Update for JCB policies and stage DA job files with Jinja2-templates (#2700) Revert PR 2681 (#2739)
NOAA-EMC · Jul 1, 2024 · 0280d2d · 0280d2d
2 parents 18d0d92 + de87067
commit 0280d2d
Show file tree

Hide file tree

Showing 44 changed files with 270 additions and 540 deletions.
diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -132,6 +132,16 @@ case ${step} in
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+"ocnanalecen")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+;;
  "ocnanalchkpt")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

diff --git a/jobs/rocoto/aeroanlfinal.sh b/jobs/rocoto/aeroanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="aeroanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/aeroanlinit.sh b/jobs/rocoto/aeroanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlrun.sh
@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlrun"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN"

diff --git a/jobs/rocoto/atmanlfinal.sh b/jobs/rocoto/atmanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="atmanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/atmanlfv3inc.sh b/jobs/rocoto/atmanlfv3inc.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT"

diff --git a/jobs/rocoto/atmanlinit.sh b/jobs/rocoto/atmanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/atmanlvar.sh b/jobs/rocoto/atmanlvar.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlvar"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL"

diff --git a/jobs/rocoto/atmensanlfinal.sh b/jobs/rocoto/atmensanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="atmensanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/atmensanlfv3inc.sh b/jobs/rocoto/atmensanlfv3inc.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT"

diff --git a/jobs/rocoto/atmensanlinit.sh b/jobs/rocoto/atmensanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/atmensanlletkf.sh b/jobs/rocoto/atmensanlletkf.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlletkf"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF"

diff --git a/jobs/rocoto/oceanice_products.sh b/jobs/rocoto/oceanice_products.sh
@@ -12,12 +12,6 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 if (( status != 0 )); then exit "${status}"; fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="oceanice_products"
 export jobid="${job}.$$"
 

diff --git a/jobs/rocoto/prepatmiodaobs.sh b/jobs/rocoto/prepatmiodaobs.sh
@@ -12,11 +12,10 @@ export job="prepatmobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow and ioda utilities
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
-PYTHONPATH="${pyiodaPATH}:${wxflowPATH}:${PYTHONPATH}"
+PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}"
 export PYTHONPATH
 
 ###############################################################

diff --git a/jobs/rocoto/prepobsaero.sh b/jobs/rocoto/prepobsaero.sh
@@ -11,12 +11,6 @@ status=$?
 export job="prepobsaero"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO"

diff --git a/jobs/rocoto/prepsnowobs.sh b/jobs/rocoto/prepsnowobs.sh
@@ -12,12 +12,11 @@ export job="prepsnowobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
 gdasappPATH="${HOMEgfs}/sorc/gdas.cd/sorc/iodaconv/src:${pyiodaPATH}"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${gdasappPATH}"
 export PYTHONPATH
 
 ###############################################################

diff --git a/jobs/rocoto/snowanl.sh b/jobs/rocoto/snowanl.sh
@@ -11,12 +11,6 @@ status=$?
 export job="snowanl"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_SNOW_ANALYSIS"

diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
@@ -29,18 +29,18 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   module load python/3.8.6
   module load crtm/2.4.0  # TODO: This is only needed when UPP_RUN=goes.  Is there a better way to handle this?
   set_trace
+
+  # Add wxflow to PYTHONPATH
+  wxflowPATH="${HOMEgfs}/ush/python"
+  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}"
+  export PYTHONPATH
+
 else
   . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="upp"
 export jobid="${job}.$$"
 

diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs
@@ -294,7 +294,6 @@ if [[ "${skip_mom6}" == "false" ]]; then
   export cplflx=".true."
   model_list="${model_list}.ocean"
   nthreads_mom6=1
-  MOM6_DIAG_MISVAL="-1e34"
   case "${mom6_res}" in
     "500")
       ntasks_mom6=8
@@ -307,6 +306,11 @@ if [[ "${skip_mom6}" == "false" ]]; then
       CHLCLIM="seawifs_1998-2006_smoothed_2X.nc"
       MOM6_RESTART_SETTING='r'
       MOM6_RIVER_RUNOFF='False'
+      if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
+        MOM6_DIAG_MISVAL="-1e34"
+      else
+        MOM6_DIAG_MISVAL="0.0"
+      fi
       eps_imesh="4.0e-1"
       MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_25L.nc"
       MOM6_ALLOW_LANDMASK_CHANGES='False'
@@ -327,8 +331,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       TOPOEDITS="ufs.topo_edits_011818.nc"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='True'
       ;;
@@ -346,8 +352,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       eps_imesh="1.0e-1"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else            
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='False'
       TOPOEDITS=""
@@ -366,8 +374,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       eps_imesh="1.0e-1"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='False'
       TOPOEDITS=""

diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl
@@ -15,14 +15,17 @@ export INTERP_METHOD='barycentric'
 if [[ ${DOHYBVAR} = "YES" ]]; then
     # shellcheck disable=SC2153
     export CASE_ANL=${CASE_ENS}
-    export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
 else
     export CASE_ANL=${CASE}
-    export BERROR_YAML="background_error_static_${STATICB_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}"
 fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2"
+export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2"
+export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2"
 
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@

diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl
@@ -12,6 +12,7 @@ export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2"
 
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -401,7 +401,6 @@ if [[ "${skip_mom6}" == "false" ]]; then
   export cplflx=".true."
   model_list="${model_list}.ocean"
   nthreads_mom6=1
-  MOM6_DIAG_MISVAL="-1e34"
   case "${mom6_res}" in
     "500")
       ntasks_mom6=8
@@ -414,6 +413,11 @@ if [[ "${skip_mom6}" == "false" ]]; then
       CHLCLIM="seawifs_1998-2006_smoothed_2X.nc"
       MOM6_RESTART_SETTING='r'
       MOM6_RIVER_RUNOFF='False'
+      if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
+        MOM6_DIAG_MISVAL="-1e34"
+      else
+        MOM6_DIAG_MISVAL="0.0"
+      fi
       eps_imesh="4.0e-1"
       MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_25L.nc"
       MOM6_ALLOW_LANDMASK_CHANGES='False'
@@ -434,8 +438,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       TOPOEDITS="ufs.topo_edits_011818.nc"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='True'
       ;;
@@ -453,8 +459,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       eps_imesh="1.0e-1"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='False'
       TOPOEDITS=""
@@ -473,8 +481,10 @@ if [[ "${skip_mom6}" == "false" ]]; then
       eps_imesh="1.0e-1"
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
+        MOM6_DIAG_MISVAL="-1e34"
       else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
+        MOM6_DIAG_MISVAL="0.0"
       fi
       MOM6_ALLOW_LANDMASK_CHANGES='False'
       TOPOEDITS=""

diff --git a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 b/parm/gdas/staging/atm_berror_gsibec.yaml.j2
@@ -0,0 +1,8 @@
+{% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %}
+
+mkdir:
+- '{{ DATA }}/berror'
+copy:
+{% for fname in fname_list %}
+- ['{{ HOMEgfs }}/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror']
+{% endfor %}