Merge branch 'develop' into fix/c768_hera

.gitignore

@@ -53,11 +53,8 @@ parm/gdas/soca
 parm/gdas/jcb-gdas
 parm/gdas/jcb-algorithms
 parm/monitor
-parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
-parm/post/gtg.config.gfs
-parm/post/gtg_imprintings.txt
 parm/post/optics_luts_DUST_nasa.dat
 parm/post/optics_luts_NITR_nasa.dat
 parm/post/optics_luts_SALT.dat
@@ -70,47 +67,14 @@ parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
-parm/post/post_tag_gfs65
-parm/post/postcntrl_gefs.xml
-parm/post/postcntrl_gefs_aerosol.xml
-parm/post/postcntrl_gefs_anl.xml
-parm/post/postcntrl_gefs_chem.xml
-parm/post/postcntrl_gefs_f00.xml
-parm/post/postcntrl_gfs.xml
-parm/post/postcntrl_gfs_anl.xml
-parm/post/postcntrl_gfs_f00.xml
-parm/post/postcntrl_gfs_f00_two.xml
-parm/post/postcntrl_gfs_flux.xml
-parm/post/postcntrl_gfs_flux_f00.xml
-parm/post/postcntrl_gfs_goes.xml
-parm/post/postcntrl_gfs_goes.xml-new
-parm/post/postcntrl_gfs_two.xml
-parm/post/postcntrl_gfs_wafs.xml
-parm/post/postcntrl_gfs_wafs_anl.xml
-parm/post/postxconfig-NT-GEFS-ANL.txt
-parm/post/postxconfig-NT-GEFS-F00.txt
-parm/post/postxconfig-NT-GEFS-F00-aerosol.txt
-parm/post/postxconfig-NT-GEFS-WAFS.txt
-parm/post/postxconfig-NT-GEFS-aerosol.txt
-parm/post/postxconfig-NT-GEFS.txt
-parm/post/postxconfig-NT-GFS-ANL.txt
-parm/post/postxconfig-NT-GFS-F00-TWO.txt
-parm/post/postxconfig-NT-GFS-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX.txt
-parm/post/postxconfig-NT-GFS-GOES.txt
-parm/post/postxconfig-NT-GFS-TWO.txt
-parm/post/postxconfig-NT-GFS-WAFS-ANL.txt
-parm/post/postxconfig-NT-GFS-WAFS.txt
-parm/post/postxconfig-NT-GFS.txt
-parm/post/postxconfig-NT-gefs-aerosol.txt
-parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/gfs
+parm/post/gefs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
-parm/ufs/model_configure_nest.IN
+parm/ufs/input_global_nest.nml.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN

env/WCOSS2.env

@@ -272,7 +272,7 @@ elif [[ "${step}" = "postsnd" ]]; then
     export OMP_NUM_THREADS=1
 
     export NTHREADS_POSTSND=${NTHREADS1}
-    export APRUN_POSTSND="${APRUN} --depth=${NTHREADS_POSTSND} --cpu-bind depth"
+    export mpmd_opt="-ppn 21 ${mpmd_opt}"
 
     export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}

parm/archive/enkf.yaml.j2

@@ -54,12 +54,14 @@ enkf:
                            "radstat.ensmean"] %}
         {% else %}
         {% if lobsdiag_forenkf %}
-        {% set da_files = ["atmens_observer.yaml",
-                           "atmens_solver.yaml",
+        {% set da_files = ["atmensanlobs.yaml",
+                           "atmensanlsol.yaml",
+                           "atmensanlfv3inc.yaml",
                            "atminc.ensmean.nc",
                            "atmensstat"] %}
         {% else %}
-        {% set da_files = ["atmens.yaml",
+        {% set da_files = ["atmensanlletkf.yaml",
+                           "atmensanlfv3inc.yaml",
                            "atminc.ensmean.nc",
                            "atmensstat"] %}
         {% endif %}

parm/archive/gdas.yaml.j2

@@ -58,7 +58,8 @@ gdas:
 
         # Analysis state
             {% if DO_JEDIATMVAR %}
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat"
             {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat"

parm/archive/gefs_arcdir.yaml.j2

@@ -0,0 +1,38 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set head = RUN + ".t" + cycle_HH + "z." %}
+
+# Declare the GEFS_ARCH where atmos data will be sent
+{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %}
+
+{% set file_set = [] %}
+
+{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                      '${RUN}':RUN,
+                      '${YMD}':cycle_YMD,
+                      '${HH}':cycle_HH,
+                      '${GRID}': '1p00',
+                      '${MEMDIR}': 'ensstat' }) %}
+
+{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+
+# Select ensstat files to copy to the arcdir
+{% if RUN == "gefs" %}
+    {% set ensstat_files = [] %}
+    {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %}
+        {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
+            {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~
+                                        "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2",
+                                        GEFS_ARCH]) %}
+        {% endfor %}
+    {% endif %}
+{% endif %}
+{% set file_set = ensstat_files %}
+# Actually write the yaml
+mkdir:
+    - "{{ GEFS_ARCH }}"
+copy:
+   {% for source_dest_pair in file_set %}
+   - {{ source_dest_pair }}
+   {% endfor %}

parm/archive/gfsa.yaml.j2

@@ -32,7 +32,8 @@ gfsa:
 
         # State data
         {% if DO_JEDIATMVAR %}
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat"

parm/config/gefs/config.base

@@ -268,10 +268,6 @@ export OUTPUT_GRID="gaussian_grid"
 export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST
 export WRITE_NSFLIP=".true."
 
-# Override normal post flat files for GEFS
-export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-GEFS.txt"
-export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-GEFS-F00.txt"
-
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
 export imp_physics=8
 

parm/config/gefs/config.ufs

@@ -254,6 +254,25 @@ export ntasks_fv3_gfs
 export ntasks_quilt
 export ntasks_quilt_gfs
 
+# Determine whether to use compression in the write grid component based on resolution
+case ${fv3_res} in
+  "C48" | "C96" | "C192" | "C384")
+    zstandard_level=0
+    ideflate=0
+    quantize_nsd=0
+    ;;
+  "C768" | "C1152" | "C3072")
+    zstandard_level=0
+    ideflate=1
+    quantize_nsd=5
+    ;;
+  *)
+    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
+    exit 15
+    ;;
+esac
+export zstandard_level ideflate quantize_nsd
+
 # Determine whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
   "C48" | "C96" | "C192" | "C384")
@@ -353,7 +372,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
         MOM6_DIAG_MISVAL="-1e34"
-      else            
+      else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
         MOM6_DIAG_MISVAL="0.0"
       fi

parm/config/gefs/yaml/defaults.yaml

@@ -14,3 +14,5 @@ base:
   FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
   USE_OCN_PERTURB_FILES: "false"
+  HPSSARCH: "NO"
+  LOCALARCH: "NO"

parm/config/gfs/config.atmensanl

@@ -6,7 +6,11 @@
 echo "BEGIN: config.atmensanl"
 
 export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
-export JCB_ALGO_YAML=@JCB_ALGO_YAML@
+if [[ ${lobsdiag_forenkf} = ".false." ]] ; then
+    export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@
+else
+    export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@
+fi    
 
 export INTERP_METHOD='barycentric'
 

parm/config/gfs/config.resources

@@ -1191,9 +1191,9 @@ case ${step} in
 
   "postsnd")
     walltime="02:00:00"
-    ntasks=40
-    threads_per_task=8
-    tasks_per_node=10
+    export ntasks=141
+    threads_per_task=6
+    export tasks_per_node=21 
     export ntasks_postsndcfp=9
     export tasks_per_node_postsndcfp=1
     postsnd_req_cores=$(( tasks_per_node * threads_per_task ))

parm/config/gfs/config.resources.ORION

@@ -9,8 +9,13 @@ case ${step} in
     # Remove this block once GSI issue is resolved
     # https://github.com/NOAA-EMC/GSI/pull/764
     # https://github.com/JCSDA/spack-stack/issues/1166
-    export wtime_anal_gdas="02:40:00"
-    export wtime_anal_gfs="02:00:00"
+    export walltime_gdas="02:40:00"
+    export walltime_gfs="02:00:00"
+  ;;
+  "eobs")
+    # TODO:
+    # Remove this block once the GSI issue is resolved.
+    export walltime="00:45:00"
   ;;
   *)
   ;;

parm/config/gfs/config.ufs

@@ -356,6 +356,25 @@ export ntasks_fv3_gfs
 export ntasks_quilt_gdas
 export ntasks_quilt_gfs
 
+# Determine whether to use compression in the write grid component based on resolution
+case ${fv3_res} in
+  "C48" | "C96" | "C192" | "C384")
+    zstandard_level=0
+    ideflate=0
+    quantize_nsd=0
+    ;;
+  "C768" | "C1152" | "C3072")
+    zstandard_level=0
+    ideflate=1
+    quantize_nsd=5
+    ;;
+  *)
+    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
+    exit 15
+    ;;
+esac
+export zstandard_level ideflate quantize_nsd
+
 # Determine whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
   "C48" | "C96" | "C192" | "C384")

parm/config/gfs/yaml/defaults.yaml

@@ -31,7 +31,8 @@ atmanl:
   IO_LAYOUT_Y: 1
 
 atmensanl:
-  JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
+  JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
+  JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
   LAYOUT_X_ATMENSANL: 8
   LAYOUT_Y_ATMENSANL: 8
   IO_LAYOUT_X: 1

parm/post/upp.yaml

@@ -18,7 +18,7 @@ analysis:
     rdaod: True
   data_in:
     copy:
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-ANL.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
@@ -32,9 +32,9 @@ forecast:
   data_in:
     copy:
       {% if forecast_hour == 0 %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-F00-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-f00-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% else %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% endif %}
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
@@ -81,10 +81,10 @@ goes:
       {% endfor %}
       - ["{{ 'CRTM_FIX' | getenv }}/AerosolCoeff.bin", "{{ DATA }}/"]
       - ["{{ 'CRTM_FIX' | getenv }}/CloudCoeff.bin", "{{ DATA }}/"]
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-GOES.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
     copy:
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"]
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"]