From 20fb88c057b22991ed3b889c7388a271cc5db99d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 28 Aug 2024 17:01:06 +0000 Subject: [PATCH] replace separate JEDI radiance bias correction files with tarball (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 ++ parm/stage/analysis.yaml.j2 | 9 +---- ush/python/pygfs/task/analysis.py | 36 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 48 ++++++++++----------------- 4 files changed, 38 insertions(+), 57 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 9d86292065..7a011671be 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -32,6 +32,8 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% else %} + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index d30389644a..4068f1e928 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,17 +10,10 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} - {% if DO_JEDIATMVAR %} - {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} - {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} - - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] - {% endfor %} - {% endfor %} - {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950..b8e7809263 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -42,8 +42,7 @@ def initialize(self) -> None: FileHandler(obs_dict).sync() # some analyses need to stage bias corrections - bias_dict = self.get_bias_dict() - FileHandler(bias_dict).sync() + self.get_bias() # link jedi executable to run directory self.link_jediexe() @@ -128,25 +127,21 @@ def get_obs_dict(self) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + def get_bias(self) -> None: + """Stage radiance bias correciton files - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- + Task: GDAS task Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ - logger.info(f"Extracting a list of bias correction files from Jedi config file") + logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -156,17 +151,22 @@ def get_bias_dict(self) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.radbcor" + copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + FileHandler(bias_dict).sync() + + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c..e32dcdf815 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -207,37 +207,23 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # copy bias correction files to ROTDIR - logger.info("Copy bias correction files from DATA/ to COM/") - biasdir = os.path.join(self.task_config.DATA, 'bc') - biasls = os.listdir(biasdir) - biaslist = [] - for bfile in biasls: - src = os.path.join(biasdir, bfile) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - biaslist.append([src, dest]) - - gprefix = f"{self.task_config.GPREFIX}" - gsuffix = f"{to_YMDH(self.task_config.previous_cycle)}" + ".txt" - aprefix = f"{self.task_config.APREFIX}" - asuffix = f"{to_YMDH(self.task_config.current_cycle)}" + ".txt" - - logger.info(f"Copying {gprefix}*{gsuffix} from DATA/ to COM/ as {aprefix}*{asuffix}") - obsdir = os.path.join(self.task_config.DATA, 'obs') - obsls = os.listdir(obsdir) - for ofile in obsls: - if ofile.endswith(".txt"): - src = os.path.join(obsdir, ofile) - tfile = ofile.replace(gprefix, aprefix) - tfile = tfile.replace(gsuffix, asuffix) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile) - biaslist.append([src, dest]) - - bias_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': biaslist, - } - FileHandler(bias_copy).sync() + # path of output radiance bias correction tarfile + bfile = f"{self.task_config.APREFIX}radbcor" + radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) + + # get lists of radiance bias correction files to put in tarball + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + + # tar radiance bias correction files to ROTDIR + logger.info(f"Creating radiance bias correction tar file {radtar}") + with tarfile.open(radtar, 'w') as radbcor: + for satfile in satlist: + radbcor.add(satfile, arcname=os.path.basename(satfile)) + for tlapfile in tlaplist: + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) + logger.info(f"Add {radbcor.getnames()}") + radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file")