Handle bias correction files for aerosol DA

jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE

@@ -24,7 +24,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \
-    COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL
+    COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL \
+    COMIN_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL
 
 ###############################################################
 # Run relevant script

parm/gdas/aero_stage_variational.yaml.j2

@@ -13,7 +13,7 @@
     {% set bkg_times = [] %}
     {% do bkg_times.append(current_cycle) %}
 {% endif %}
-{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %}
+{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.', 'sfc_data.', 'fv_srf_wnd.res.'] %}
 ######################################
 mkdir:
 - "{{ DATA }}/anl"

parm/stage/aero.yaml.j2

@@ -0,0 +1,17 @@
+aero:
+    mkdir:
+        {% for mem in range(first_mem, last_mem + 1) %}
+        {% set imem = mem - first_mem %}
+        {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %}
+        - "{{ COMOUT_CHEM_ANALYSIS_MEM }}"
+        {% endfor %}
+    link_req:
+        {% for mem in range(first_mem, last_mem + 1) %}
+        {% set imem = mem - first_mem %}
+        {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %}
+        {% for ftype in ["aero_varbc_params.tar"] %}
+        {% if path_exists(ICSDIR ~ "/" ~ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %}
+        - ["{{ ICSDIR }}/{{ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_CHEM_ANALYSIS_MEM }}"]
+        {% endif %}
+        {% endfor %}
+        {% endfor %} # mem loop

parm/stage/master_gfs.yaml.j2

@@ -78,6 +78,7 @@
 {% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %}
 {% set COMOUT_MED_RESTART_PREV_MEM_list = [] %}
 {% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %}
+{% set COMOUT_CHEM_ANALYSIS_MEM_list = [] %}
 
 # Construct member COM directory lists
 # ------------------------------------
@@ -110,6 +111,7 @@
     {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %}
     {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %}
     {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %}
+    {% set COMOUT_CHEM_ANALYSIS_MEM = COM_CHEM_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %}
 
     # Append the member COM directories
     {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%}
@@ -122,6 +124,7 @@
     {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%}
     {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%}
     {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%}
+    {% do COMOUT_CHEM_ANALYSIS_MEM_list.append(COMOUT_CHEM_ANALYSIS_MEM)%}
 
 {% endfor %}
 
@@ -189,3 +192,9 @@
 {% include "wave.yaml.j2" %}
 {% endfilter %}
 {% endif %}
+
+{% if DO_AERO_ANL %}
+{% filter indent(width=4) %}
+{% include "aero.yaml.j2" %}
+{% endfilter %}
+{% endif %}

scripts/exglobal_stage_ic.py

@@ -23,7 +23,7 @@ def main():
             'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR',
             'OCNRES', 'waveGRD', 'ntiles', 'DOIAU',
             'DO_JEDIOCNVAR', 'DO_STARTMEM_FROM_JEDIICE',
-            'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_CA',
+            'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_CA', 'DO_AERO_ANL',
             'USE_ATM_ENS_PERTURB_FILES', 'USE_OCN_ENS_PERTURB_FILES']
 
     stage_dict = AttrDict()

ush/python/pygfs/task/aero_analysis.py

@@ -101,17 +101,21 @@ def initialize(self) -> None:
         logger.debug(f"Observation files:\n{pformat(obs_dict)}")
 
         # # stage bias corrections
-        # logger.info(f"Staging list of bias correction files")
-        # bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging')
-        # if bias_dict['copy'] is None:
-        #     logger.info(f"No bias correction files to stage")
-        # else:
-        #     bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy'])
-        #     FileHandler(bias_dict).sync()
-        #     logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
-
-        #     # extract bias corrections
-        #     Jedi.extract_tar_from_filehandler_dict(bias_dict)
+        logger.info(f"Staging list of bias correction files")
+        bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging')
+
+        if bias_dict['copy'] is None:
+            logger.info(f"No bias correction files to stage")
+        else:
+            try:
+                bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy'])
+                FileHandler(bias_dict).sync()
+                logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
+
+                # extract bias corrections
+                Jedi.extract_tar_from_filehandler_dict(bias_dict)
+            except FileNotFoundError:
+                logger.error(f"Bias correction files or directories do not exist:\n{pformat(bias_dict)}")
 
         # stage CRTM fix files
         logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")
@@ -183,13 +187,24 @@ def finalize(self) -> None:
         self._add_fms_cube_sphere_increments()
 
         # tar up bias correction files
-        # NOTE TODO
+        bfile = f"{self.task_config.APREFIX}aero_varbc_params.tar"
+        aertar = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, bfile)
+
+        # get lists of aerosol bias correction files to add to tarball
+        satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc'))
 
         # copy files back to COM
         logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}")
         aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config)
         FileHandler(aero_var_final_list).sync()
 
+        # tar aerosol bias correction files to ROTDIR
+        logger.info(f"Creating aerosol bias correction tar file {aertar}")
+        with tarfile.open(aertar, 'w') as aerbcor:
+            for satfile in satlist:
+                aerbcor.add(satfile, arcname=os.path.basename(satfile))
+            logger.info(f"Add {aerbcor.getnames()}")
+
         # open tar file for writing
         with tarfile.open(aerostat, "w|gz") as archive:
             for diagfile in diags: