Add an archive task to GEFS system to archive files in HPSS

parm/archive/gefs_extracted_atmos.yaml.j2

@@ -0,0 +1,86 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set head = RUN + ".t" + cycle_HH + "z." %}
+
+gefs_atmos:
+    name: "GEFS_ATMOS"
+    target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs_atmos.tar"
+    required:
+#select mem%03d and ensstat files required
+{% set members = ["ensstat"] %}
+{% for mem_nm in range(0, NMEM_ENS + 1) %}
+    {% do members.append("mem" ~ '%03d' % mem_nm ) %}
+{% endfor %}
+
+{% if REPLAY_ICS %}
+    {% set ofst_hr = OFFSET_START_HOUR %}
+{% else %}
+    {% set ofst_hr = FHMIN_GFS %}
+{% endif %}
+
+{% for mem in members %}
+    {% for res in ['0p25', '0p50', '1p00'] %}
+        {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                              '${RUN}':RUN,
+                              '${YMD}':cycle_YMD,
+                              '${HH}':cycle_HH,
+                              '${GRID}':res,
+                              '${MEMDIR}':mem }) %}
+
+        {% set COMIN_ATMOS_GRIB = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+
+# Select pgrb and grib files to copy to the atardir
+        {% if path_exists(COMIN_ATMOS_GRIB) %}
+            {% if FHMAX_HF_GFS  == 0 %}
+                {% for fhr in range(ofst_hr, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
+                    {% if mem=="ensstat" %}
+                        {% set file_name = head ~ "mean.pres_." ~ res ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2" %}
+                        {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                        - "{{ file_path | relpath(ROTDIR)}}"
+                    {% else %}
+                        {% set file_name = head ~ "pgrb2." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                        {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                        - "{{ file_path | relpath(ROTDIR)}}"
+                        {% set file_name = head ~ "pgrb2b." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                        {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                        - "{{ file_path | relpath(ROTDIR)}}"
+                    {% endif %}
+                {% endfor %}
+            {% else %}
+                 {% if res == "0p25" %}
+                    {% for fhr in range(ofst_hr, FHMAX_HF_GFS + FHOUT_HF_GFS, FHOUT_HF_GFS) %}
+                        {% if mem=="ensstat" %}
+                            {% set file_name = head ~ "mean.pres_." ~ res ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2" %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                        {% else %}
+                            {% set file_name = head ~ "pgrb2." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                            {% set file_name = head ~ "pgrb2b." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                        {% endif %}
+                    {% endfor %}
+                {% endif %}
+                {% if res == "0p50" %}
+                    {% for fhr in range(FHMAX_HF_GFS + FHOUT_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
+                        {% if mem=="ensstat" %}
+                            {% set file_name = head ~ "mean.pres_." ~ res ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2" %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                        {% else %}
+                            {% set file_name = head ~ "pgrb2." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                            {% set file_name = head ~ "pgrb2b." ~ res ~ ".f" ~ '%03d'|format(fhr) %}
+                            {% set file_path = COMIN_ATMOS_GRIB ~ "/" ~ file_name %}
+                            - "{{ file_path | relpath(ROTDIR)}}"
+                        {% endif %}
+                    {% endfor %}
+                {% endif %}
+            {% endif %}
+        {% endif %}
+    {% endfor %}
+{% endfor %}

parm/archive/gefs_extracted_ice.yaml.j2

@@ -0,0 +1,33 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set head = RUN + ".ice.t" + cycle_HH + "z." %}
+
+gefs_ice:
+    name: "GEFS_ICE"
+    target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs_ice.tar"
+    required:
+#select mem%03d and ensstat files required
+{% set members = [] %}
+{% for mem_nm in range(0, NMEM_ENS + 1) %}
+    {% do members.append("mem" ~ '%03d' % mem_nm ) %}
+{% endfor %}
+
+{% for mem in members %}
+    {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                          '${RUN}':RUN,
+                          '${YMD}':cycle_YMD,
+                          '${HH}':cycle_HH,
+                          '${MEMDIR}':mem }) %}
+
+    {% set COMIN_ICE_HISTORY = COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) %}
+
+# Select netcdf files to copy to the atardir
+    {% if path_exists(COMIN_ICE_HISTORY) %}
+        {% for fhr in range(FHMIN_GFS + FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %}
+            {% set file_name = head ~ FHOUT_ICE_GFS ~ "hr_avg" ~ ".f" ~ '%03d'|format(fhr) ~ ".nc" %}
+            {% set file_path = COMIN_ICE_HISTORY ~ "/" ~ file_name %}
+            - "{{ file_path | relpath(ROTDIR)}}"
+        {% endfor %}
+    {% endif %}
+{% endfor %}

parm/archive/gefs_extracted_ocean.yaml.j2

@@ -0,0 +1,36 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set head = RUN + ".ocean.t" + cycle_HH + "z." %}
+
+gefs_ocean:
+    name: "GEFS_OCEAN"
+    target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs_ocean.tar"
+    required:
+#select mem%03d and ensstat files required
+{% set members = [] %}
+{% for mem_nm in range(0, NMEM_ENS + 1) %}
+    {% do members.append("mem" ~ '%03d' % mem_nm ) %}
+{% endfor %}
+
+{% set res = (OCNRES|string())[0] ~ "p" ~ (OCNRES|string())[-2:] %}
+
+{% for mem in members %}
+    {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                          '${RUN}':RUN,
+                          '${YMD}':cycle_YMD,
+                          '${HH}':cycle_HH,
+                          '${GRID}':res,
+                          '${MEMDIR}':mem }) %}
+
+    {% set COMIN_OCEAN_NETCDF = COM_OCEAN_NETCDF_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+
+    # Select netcdf files to copy to the atardir
+    {% if path_exists(COMIN_OCEAN_NETCDF) %}
+        {% for fhr in range(FHMIN_GFS + FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %}
+            {% set file_name = head ~ res ~ ".f" ~ '%03d'|format(fhr) ~ ".nc" %}
+            {% set file_path = COMIN_OCEAN_NETCDF ~ "/" ~ file_name %}
+            - "{{ file_path | relpath(ROTDIR)}}"
+        {% endfor %}
+    {% endif %}
+{% endfor %}

parm/archive/gefs_extracted_wave.yaml.j2

@@ -0,0 +1,41 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set head = RUN + "wave.t" + cycle_HH + "z." %}
+
+gefs_wave:
+    name: "GEFS_WAVE"
+    target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gefs_wave.tar"
+    required:
+{% if REPLAY_ICS %}
+    {% set ofst_hr = OFFSET_START_HOUR %}
+{% else %}
+    {% set ofst_hr = FHMIN_GFS %}
+{% endif %}
+
+{% set res = (waveGRD[-3:])[0] ~ "p" ~ (waveGRD[-3:])[-2:] %}
+
+#select mem%03d and ensstat files required
+{% set members = [] %}
+{% for mem_nm in range(0, NMEM_ENS + 1) %}
+    {% do members.append("mem" ~ '%03d' % mem_nm ) %}
+{% endfor %}
+
+{% for mem in members %}
+    {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                          '${RUN}':RUN,
+                          '${YMD}':cycle_YMD,
+                          '${HH}':cycle_HH,
+                          '${MEMDIR}':mem }) %}
+
+    {% set COMIN_WAVE_GRID = COM_WAVE_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+
+    # Select grib2 files to copy to the atardir
+    {% if path_exists(COMIN_WAVE_GRID) %}
+        {% for fhr in range(ofst_hr, FHMAX_GFS + FHOUT_HF_WAV, FHOUT_HF_WAV) %}
+            {% set file_name = head ~ "global." ~ res ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2" %}
+            {% set file_path = COMIN_WAVE_GRID ~ "/" ~ file_name %}
+            - "{{ file_path | relpath(ROTDIR)}}"
+        {% endfor %}
+    {% endif %}
+{% endfor %}

parm/archive/master_gefs.yaml.j2

@@ -0,0 +1,12 @@
+# Set variables/lists needed to parse the gefs templates
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+
+datasets:
+{% filter indent(width=4) %}
+    {% include "gefs_extracted_atmos.yaml.j2" %}
+    {% include "gefs_extracted_ocean.yaml.j2" %}
+    {% include "gefs_extracted_ice.yaml.j2" %}
+    {% include "gefs_extracted_wave.yaml.j2" %}
+{% endfilter %}

parm/config/gefs/yaml/defaults.yaml

@@ -14,5 +14,4 @@ base:
   FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
   USE_OCN_PERTURB_FILES: "false"
-  HPSSARCH: "NO"
   LOCALARCH: "NO"

parm/config/gfs/config.com

@@ -84,6 +84,7 @@ declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input'
 declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean'
 declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean'
 declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf'
+declare -rx COM_OCEAN_NETCDF_GRID_TMPL=${COM_OCEAN_NETCDF_TMPL}'/${GRID}'
 declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2'
 declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}'
 

scripts/exglobal_archive.py

@@ -28,10 +28,11 @@ def main():
             'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC',
             'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV',
             'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS',
-            'restart_interval_gdas', 'restart_interval_gfs',
+            'restart_interval_gdas', 'restart_interval_gfs', 'DO_EXTRACTVARS',
             'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR',
-            'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS',
-            'IAUFHRS', 'DO_FIT2OBS', 'NET']
+            'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', 'waveGRD',
+            'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', 'REPLAY_ICS',
+            'OFFSET_START_HOUR']
 
     archive_dict = AttrDict()
     for key in keys:

ush/python/pygfs/task/archive.py

@@ -114,9 +114,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str
             self.tar_cmd = ""
             return arcdir_set, []
 
-        if arch_dict.NET == "gefs":
-            raise NotImplementedError("GEFS archiving is not yet implemented!")
-
         master_yaml = "master_" + arch_dict.RUN + ".yaml.j2"
 
         parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml),