replace separate JEDI radiance bias correction files with tarball (NO…

…AA-EMC#2862)

parm/archive/gdas_restarta.yaml.j2

@@ -32,6 +32,8 @@ gdas_restarta:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt"
+        {% else %}
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor"
         {% endif %}
 
         # Snow surface data

parm/stage/analysis.yaml.j2

@@ -10,17 +10,10 @@ analysis:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}
-        {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %}
+        {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %}
         {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"]
         {% endif %}
         {% endfor %}
-        {% if DO_JEDIATMVAR %}
-        {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %}
-        {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %}
-        - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"]
-        {% endfor %}
-        {% endfor %}
-        {% endif %}
         {% endfor %} # mem loop
 {% endif %}

ush/python/pygfs/task/analysis.py

@@ -42,8 +42,7 @@ def initialize(self) -> None:
         FileHandler(obs_dict).sync()
 
         # some analyses need to stage bias corrections
-        bias_dict = self.get_bias_dict()
-        FileHandler(bias_dict).sync()
+        self.get_bias()
 
         # link jedi executable to run directory
         self.link_jediexe()
@@ -128,25 +127,21 @@ def get_obs_dict(self) -> Dict[str, Any]:
         return obs_dict
 
     @logit(logger)
-    def get_bias_dict(self) -> Dict[str, Any]:
-        """Compile a dictionary of observation files to copy
+    def get_bias(self) -> None:
+        """Stage radiance bias correciton files
 
-        This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of
-        observation bias correction files that are to be copied to the run directory
-        from the component directory.
-        TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in
-        `analysis.py` and should be implemented in the component where this is applicable.
+        This method stages radiance bias correction files in the obs sub-diretory of the run directory
 
         Parameters
         ----------
+        Task: GDAS task
 
         Returns
         ----------
-        bias_dict: Dict
-            a dictionary containing the list of observation bias files to copy for FileHandler
+        None
         """
 
-        logger.info(f"Extracting a list of bias correction files from Jedi config file")
+        logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances")
         observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations')
         logger.debug(f"observations:\n{pformat(observations)}")
 
@@ -156,17 +151,22 @@ def get_bias_dict(self) -> Dict[str, Any]:
                 obfile = ob['obs bias']['input file']
                 obdir = os.path.dirname(obfile)
                 basename = os.path.basename(obfile)
-                prefix = '.'.join(basename.split('.')[:-2])
-                for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']:
-                    bfile = f"{prefix}.{file}"
-                    copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
-                    # TODO: Why is this specific to ATMOS?
+                prefix = '.'.join(basename.split('.')[:-3])
+                bfile = f"{prefix}.radbcor"
+                copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
+                break
 
         bias_dict = {
             'mkdir': [os.path.join(self.task_config.DATA, 'bc')],
             'copy': copylist
         }
-        return bias_dict
+        FileHandler(bias_dict).sync()
+
+        radtar = os.path.join(obdir, bfile)
+        with tarfile.open(radtar, "r") as radbcor:
+            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
+            logger.info(f"Extract {radbcor.getnames()}")
+        radbcor.close()
 
     @logit(logger)
     def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None:

ush/python/pygfs/task/atm_analysis.py

@@ -207,37 +207,23 @@ def finalize(self: Analysis) -> None:
         }
         FileHandler(yaml_copy).sync()
 
-        # copy bias correction files to ROTDIR
-        logger.info("Copy bias correction files from DATA/ to COM/")
-        biasdir = os.path.join(self.task_config.DATA, 'bc')
-        biasls = os.listdir(biasdir)
-        biaslist = []
-        for bfile in biasls:
-            src = os.path.join(biasdir, bfile)
-            dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile)
-            biaslist.append([src, dest])
-
-        gprefix = f"{self.task_config.GPREFIX}"
-        gsuffix = f"{to_YMDH(self.task_config.previous_cycle)}" + ".txt"
-        aprefix = f"{self.task_config.APREFIX}"
-        asuffix = f"{to_YMDH(self.task_config.current_cycle)}" + ".txt"
-
-        logger.info(f"Copying {gprefix}*{gsuffix} from DATA/ to COM/ as {aprefix}*{asuffix}")
-        obsdir = os.path.join(self.task_config.DATA, 'obs')
-        obsls = os.listdir(obsdir)
-        for ofile in obsls:
-            if ofile.endswith(".txt"):
-                src = os.path.join(obsdir, ofile)
-                tfile = ofile.replace(gprefix, aprefix)
-                tfile = tfile.replace(gsuffix, asuffix)
-                dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile)
-                biaslist.append([src, dest])
-
-        bias_copy = {
-            'mkdir': [self.task_config.COM_ATMOS_ANALYSIS],
-            'copy': biaslist,
-        }
-        FileHandler(bias_copy).sync()
+        # path of output radiance bias correction tarfile
+        bfile = f"{self.task_config.APREFIX}radbcor"
+        radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile)
+
+        # get lists of radiance bias correction files to put in tarball
+        satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc'))
+        tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt'))
+
+        # tar radiance bias correction files to ROTDIR
+        logger.info(f"Creating radiance bias correction tar file {radtar}")
+        with tarfile.open(radtar, 'w') as radbcor:
+            for satfile in satlist:
+                radbcor.add(satfile, arcname=os.path.basename(satfile))
+            for tlapfile in tlaplist:
+                radbcor.add(tlapfile, arcname=os.path.basename(tlapfile))
+            logger.info(f"Add {radbcor.getnames()}")
+        radbcor.close()
 
         # Copy FV3 atm increment to comrot directory
         logger.info("Copy UFS model readable atm increment file")