update pygfs jedi class and atmospheric tasks to extract JEDI radianc…

…e bias correction files from tarball (NOAA-EMC#2862)

ush/python/pygfs/jedi/jedi.py

@@ -216,11 +216,10 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]:
                 obfile = ob['obs bias']['input file']
                 obdir = os.path.dirname(obfile)
                 basename = os.path.basename(obfile)
-                prefix = '.'.join(basename.split('.')[:-2])
-                for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']:
-                    bfile = f"{prefix}.{file}"
-                    copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
-                    # TODO: Why is this specific to ATMOS?
+                prefix = '.'.join(basename.split('.')[:-3])
+                bfile = f"{prefix}.rad_varbc_params.tar"
+                copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
+                break
 
         bias_dict = {
             'mkdir': [os.path.join(task_config.DATA, 'bc')],

ush/python/pygfs/task/analysis.py

@@ -41,9 +41,6 @@ def initialize(self) -> None:
         obs_dict = self.get_obs_dict()
         FileHandler(obs_dict).sync()
 
-        # some analyses need to stage bias corrections
-        self.get_bias()
-
         # link jedi executable to run directory
         self.link_jediexe()
 
@@ -126,48 +123,6 @@ def get_obs_dict(self) -> Dict[str, Any]:
         }
         return obs_dict
 
-    @logit(logger)
-    def get_bias(self) -> None:
-        """Stage radiance bias correciton files
-
-        This method stages radiance bias correction files in the obs sub-diretory of the run directory
-
-        Parameters
-        ----------
-        Task: GDAS task
-
-        Returns
-        ----------
-        None
-        """
-
-        logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances")
-        observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations')
-        logger.debug(f"observations:\n{pformat(observations)}")
-
-        copylist = []
-        for ob in observations['observers']:
-            if 'obs bias' in ob.keys():
-                obfile = ob['obs bias']['input file']
-                obdir = os.path.dirname(obfile)
-                basename = os.path.basename(obfile)
-                prefix = '.'.join(basename.split('.')[:-3])
-                bfile = f"{prefix}.rad_varbc_params.tar"
-                copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)])
-                break
-
-        bias_dict = {
-            'mkdir': [os.path.join(self.task_config.DATA, 'bc')],
-            'copy': copylist
-        }
-        FileHandler(bias_dict).sync()
-
-        radtar = os.path.join(obdir, bfile)
-        with tarfile.open(radtar, "r") as radbcor:
-            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
-            logger.info(f"Extract {radbcor.getnames()}")
-        radbcor.close()
-
     @logit(logger)
     def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None:
         """Add cubed-sphere increments to cubed-sphere backgrounds

ush/python/pygfs/task/atm_analysis.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import re
 import glob
 import gzip
 import tarfile
@@ -143,6 +144,21 @@ def initialize_analysis(self) -> None:
         FileHandler(bias_dict).sync()
         logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
 
+        # Extract radiance bias correction files from tarball
+        for action, filelist in bias_dict.items():
+            if 'copy' in action:
+                for sublist in filelist:
+                    if len(sublist) != 2:
+                        raise Exception(
+                            f"List must be of the form ['src', 'dest'], not {sublist}")
+                    src = sublist[0]
+                    if re.search(".tar", src):
+                        radtar = src
+                        with tarfile.open(radtar, "r") as radbcor:
+                            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
+                            logger.info(f"Extract {radbcor.getnames()}")
+                        radbcor.close()
+
         # stage CRTM fix files
         logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")
         crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config)

ush/python/pygfs/task/atmens_analysis.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import re
 import glob
 import gzip
 import tarfile
@@ -142,6 +143,21 @@ def initialize_analysis(self) -> None:
         FileHandler(bias_dict).sync()
         logger.debug(f"Bias correction files:\n{pformat(bias_dict)}")
 
+        # Extract radiance bias correction files from tarball
+        for action, filelist in bias_dict.items():
+            if 'copy' in action:
+                for sublist in filelist:
+                    if len(sublist) != 2:
+                        raise Exception(
+                            f"List must be of the form ['src', 'dest'], not {sublist}")
+                    src = sublist[0]
+                    if re.search(".tar", src):
+                        radtar = src
+                        with tarfile.open(radtar, "r") as radbcor:
+                            radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs'))
+                            logger.info(f"Extract {radbcor.getnames()}")
+                        radbcor.close()
+
         # stage CRTM fix files
         logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}")
         crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config)