Update for JCB policies and stage DA job files with Jinja2-templates (#…

…2700) This PR updates the `gdas.cd` hash to bring in new JCB conventions. Resolves #2699 From #2654 This PR will move much of the staging code that take place in the python initialization subroutines of the variational and ensemble DA jobs into Jinja2-templated YAML files to be passed into the wxflow file handler. Much of the staging has already been done this way, but this PR simply expands that strategy. The old Python routines that were doing this staging are now removed. This is part of a broader refactoring of the pygfs tasking. wxflow PR [#30](NOAA-EMC/wxflow#30) is a companion to this PR. Co-authored-by: danholdaway <[email protected]> Co-authored-by: DavidNew-NOAA <[email protected]> Co-authored-by: DavidNew-NOAA <[email protected]> Co-authored-by: Dan Holdaway <[email protected]>
NOAA-EMC · Jul 1, 2024 · de87067 · de87067
1 parent c49e4ee
commit de87067
Show file tree

Hide file tree

Showing 42 changed files with 248 additions and 538 deletions.
diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -132,6 +132,16 @@ case ${step} in
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+"ocnanalecen")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+;;
  "ocnanalchkpt")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

diff --git a/jobs/rocoto/aeroanlfinal.sh b/jobs/rocoto/aeroanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="aeroanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/aeroanlinit.sh b/jobs/rocoto/aeroanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlrun.sh
@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlrun"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN"

diff --git a/jobs/rocoto/atmanlfinal.sh b/jobs/rocoto/atmanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="atmanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/atmanlfv3inc.sh b/jobs/rocoto/atmanlfv3inc.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT"

diff --git a/jobs/rocoto/atmanlinit.sh b/jobs/rocoto/atmanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/atmanlvar.sh b/jobs/rocoto/atmanlvar.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmanlvar"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL"

diff --git a/jobs/rocoto/atmensanlfinal.sh b/jobs/rocoto/atmensanlfinal.sh
@@ -11,11 +11,6 @@ status=$?
 export job="atmensanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE"

diff --git a/jobs/rocoto/atmensanlfv3inc.sh b/jobs/rocoto/atmensanlfv3inc.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT"

diff --git a/jobs/rocoto/atmensanlinit.sh b/jobs/rocoto/atmensanlinit.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE"

diff --git a/jobs/rocoto/atmensanlletkf.sh b/jobs/rocoto/atmensanlletkf.sh
@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlletkf"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF"

diff --git a/jobs/rocoto/oceanice_products.sh b/jobs/rocoto/oceanice_products.sh
@@ -12,12 +12,6 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 if (( status != 0 )); then exit "${status}"; fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="oceanice_products"
 export jobid="${job}.$$"
 

diff --git a/jobs/rocoto/prepatmiodaobs.sh b/jobs/rocoto/prepatmiodaobs.sh
@@ -12,11 +12,10 @@ export job="prepatmobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow and ioda utilities
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
-PYTHONPATH="${pyiodaPATH}:${wxflowPATH}:${PYTHONPATH}"
+PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}"
 export PYTHONPATH
 
 ###############################################################

diff --git a/jobs/rocoto/prepobsaero.sh b/jobs/rocoto/prepobsaero.sh
@@ -11,12 +11,6 @@ status=$?
 export job="prepobsaero"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO"

diff --git a/jobs/rocoto/prepsnowobs.sh b/jobs/rocoto/prepsnowobs.sh
@@ -12,12 +12,11 @@ export job="prepsnowobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
 gdasappPATH="${HOMEgfs}/sorc/gdas.cd/sorc/iodaconv/src:${pyiodaPATH}"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${gdasappPATH}"
 export PYTHONPATH
 
 ###############################################################

diff --git a/jobs/rocoto/snowanl.sh b/jobs/rocoto/snowanl.sh
@@ -11,12 +11,6 @@ status=$?
 export job="snowanl"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_SNOW_ANALYSIS"

diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
@@ -29,18 +29,18 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   module load python/3.8.6
   module load crtm/2.4.0  # TODO: This is only needed when UPP_RUN=goes.  Is there a better way to handle this?
   set_trace
+
+  # Add wxflow to PYTHONPATH
+  wxflowPATH="${HOMEgfs}/ush/python"
+  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}"
+  export PYTHONPATH
+
 else
   . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="upp"
 export jobid="${job}.$$"
 

diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl
@@ -15,14 +15,17 @@ export INTERP_METHOD='barycentric'
 if [[ ${DOHYBVAR} = "YES" ]]; then
     # shellcheck disable=SC2153
     export CASE_ANL=${CASE_ENS}
-    export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
 else
     export CASE_ANL=${CASE}
-    export BERROR_YAML="background_error_static_${STATICB_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}"
 fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2"
+export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2"
+export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2"
 
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@

diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl
@@ -12,6 +12,7 @@ export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2"
 
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@

diff --git a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 b/parm/gdas/staging/atm_berror_gsibec.yaml.j2
@@ -0,0 +1,8 @@
+{% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %}
+
+mkdir:
+- '{{ DATA }}/berror'
+copy:
+{% for fname in fname_list %}
+- ['{{ HOMEgfs }}/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror']
+{% endfor %}
diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2
@@ -0,0 +1,32 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': RUN, 
+                         '${YMD}': current_cycle | to_YMD,	
+                         '${HH}': current_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- '{{ DATA }}/bkg/{{ memchar }}'
+- '{{ DATA }}/anl/{{ memchar }}'
+- '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}'
+{% endfor %}
+copy:
+{% for time in time_list %}
+   {% for imem in range(1,NMEM_ENS+1) %}
+      {% set memchar = 'mem%03d' | format(imem) %}
+      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+      	      	      	    '${RUN}': 'enkfgdas',
+                            '${YMD}': previous_cycle | to_YMD,	
+                            '${HH}': previous_cycle | strftime('%H'),
+                            '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/']
+      {% for ftype in ftype_list %}
+         {% for itile in range(1,7) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/']
+         {% endfor %}
+      {% endfor %}
+   {% endfor %}
+{% endfor %}
diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2
@@ -0,0 +1,14 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+- '{{ DATA }}/bkg'
+copy:
+{% for time in time_list %}
+- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/']
+   {% for ftype in ftype_list %}
+      {% for itile in range(1,ntiles+1) %}
+- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/']
+      {% endfor %}
+   {% endfor %}
+{% endfor %}
diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2
@@ -0,0 +1,24 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+- '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}'
+{% endfor %}
+copy:
+{% for time in time_list %}
+   {% for imem in range(1,NMEM_ENS+1) %}
+      {% set memchar = 'mem%03d' | format(imem) %}
+      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                            '${RUN}': 'enkfgdas',
+                            '${YMD}': previous_cycle | to_YMD,
+                            '${HH}': previous_cycle | strftime('%H'),
+                            '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/']
+      {% for ftype in ftype_list %}
+         {% for itile in range(1,ntiles+1) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/']
+         {% endfor %}
+      {% endfor %}
+   {% endfor %}
+{% endfor %}
diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
@@ -14,13 +14,13 @@ rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
 # Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days
 # purge_every_days should be a positive integer
-purge_every_days=3
+#purge_every_days=3
 
 # Find and delete files older than ${purge_every_days} days
-find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
+#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
 
 # Find and delete directories older than ${purge_every_days} days
-find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
+#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
 
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################

diff --git a/scripts/exglobal_prep_snow_obs.py b/scripts/exglobal_prep_snow_obs.py
@@ -21,5 +21,5 @@
     # Instantiate the snow prepare task
     SnowAnl = SnowAnalysis(config)
     SnowAnl.prepare_GTS()
-    if f"{ SnowAnl.runtime_config.cyc }" == '18':
+    if f"{ SnowAnl.task_config.cyc }" == '18':
         SnowAnl.prepare_IMS()
diff --git a/sorc/gdas.cd b/sorc/gdas.cd
diff --git a/sorc/wxflow b/sorc/wxflow