diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 62dcb517ca..dace86c4fe 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -216,11 +216,10 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.rad_varbc_params.tar" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index d9cd358219..1d8b38483b 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -41,9 +41,6 @@ def initialize(self) -> None: obs_dict = self.get_obs_dict() FileHandler(obs_dict).sync() - # some analyses need to stage bias corrections - self.get_bias() - # link jedi executable to run directory self.link_jediexe() @@ -126,48 +123,6 @@ def get_obs_dict(self) -> Dict[str, Any]: } return obs_dict - @logit(logger) - def get_bias(self) -> None: - """Stage radiance bias correciton files - - This method stages radiance bias correction files in the obs sub-diretory of the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - - logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") - observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - break - - bias_dict = { - 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], - 'copy': copylist - } - FileHandler(bias_dict).sync() - - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() - @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: """Add cubed-sphere increments to cubed-sphere backgrounds diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ad082063bb..36f768fa13 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -143,6 +144,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 55e72702b1..7004bf059c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -142,6 +143,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config)