From 4098f108a6e48df1822b1da42e9ad307845e5ef5 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Mon, 9 Sep 2024 10:58:38 -0400 Subject: [PATCH 1/5] Remove the GTS BUFR2IODA part of the snow obs prep job and Replace it with a direct read from BUFR at runtime. --- parm/config/gfs/config.prepsnowobs | 3 - parm/config/gfs/config.snowanl | 1 + scripts/exglobal_prep_snow_obs.py | 1 - ush/python/pygfs/task/snow_analysis.py | 92 ++++---------------------- 4 files changed, 15 insertions(+), 82 deletions(-) diff --git a/parm/config/gfs/config.prepsnowobs b/parm/config/gfs/config.prepsnowobs index 60ca16ce9e..20bdd89ddf 100644 --- a/parm/config/gfs/config.prepsnowobs +++ b/parm/config/gfs/config.prepsnowobs @@ -8,11 +8,8 @@ echo "BEGIN: config.prepsnowobs" # Get task specific resources . "${EXPDIR}/config.resources" prepsnowobs -export GTS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_gts.yaml.j2" export IMS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_ims.yaml.j2" -export BUFR2IODAX="${EXECgfs}/bufr2ioda.x" - export CALCFIMSEXE="${EXECgfs}/calcfIMS.exe" export FIMS_NML_TMPL="${PARMgfs}/gdas/snow/prep/fims.nml.j2" diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index a2984f190b..c1a99f99ea 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -9,6 +9,7 @@ echo "BEGIN: config.snowanl" source "${EXPDIR}/config.resources" snowanl export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2" +export GTS_LIST="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" # Name of the JEDI executable and its yaml template export JEDIEXE="${EXECgfs}/gdas.x" diff --git a/scripts/exglobal_prep_snow_obs.py b/scripts/exglobal_prep_snow_obs.py index a6a9070151..aa1eb1bb7d 100755 --- a/scripts/exglobal_prep_snow_obs.py +++ b/scripts/exglobal_prep_snow_obs.py @@ -20,6 +20,5 @@ # Instantiate the snow prepare task SnowAnl = SnowAnalysis(config) - SnowAnl.prepare_GTS() if SnowAnl.task_config.cyc == 0: SnowAnl.prepare_IMS() diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9656b00a8e..ca2d4fd49b 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -54,83 +54,6 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - @logit(logger) - def prepare_GTS(self) -> None: - """Prepare the GTS data for a global snow analysis - - This method will prepare GTS data for a global snow analysis using JEDI. - This includes: - - processing GTS bufr snow depth observation data to IODA format - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - - # create a temporary dict of all keys needed in this method - localconf = AttrDict() - keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', - 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] - for key in keys: - localconf[key] = self.task_config[key] - - # Read and render the GTS_OBS_LIST yaml - logger.info(f"Reading {self.task_config.GTS_OBS_LIST}") - prep_gts_config = parse_j2yaml(self.task_config.GTS_OBS_LIST, localconf) - logger.debug(f"{self.task_config.GTS_OBS_LIST}:\n{pformat(prep_gts_config)}") - - # copy the GTS obs files from COM_OBS to DATA/obs - logger.info("Copying GTS obs for bufr2ioda.x") - FileHandler(prep_gts_config.gtsbufr).sync() - - logger.info("Link BUFR2IODAX into DATA/") - exe_src = self.task_config.BUFR2IODAX - exe_dest = os.path.join(localconf.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - # Create executable instance - exe = Executable(self.task_config.BUFR2IODAX) - - def _gtsbufr2iodax(exe, yaml_file): - if not os.path.isfile(yaml_file): - logger.exception(f"FATAL ERROR: {yaml_file} not found") - raise FileNotFoundError(yaml_file) - - logger.info(f"Executing {exe}") - try: - exe(yaml_file) - except OSError: - raise OSError(f"Failed to execute {exe} {yaml_file}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}") - - # Loop over entries in prep_gts_config.bufr2ioda keys - # 1. generate bufr2ioda YAML files - # 2. execute bufr2ioda.x - for name in prep_gts_config.bufr2ioda.keys(): - gts_yaml = os.path.join(self.task_config.DATA, f"bufr_{name}_snow.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") - temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) - save_as_yaml(temp_yaml, gts_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") - - # execute BUFR2IODAX to convert {name} bufr data into IODA format - _gtsbufr2iodax(exe, gts_yaml) - - # Ensure the IODA snow depth GTS file is produced by the IODA converter - # If so, copy to COM_OBS/ - try: - FileHandler(prep_gts_config.gtsioda).sync() - except OSError as err: - logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce GTS ioda files") - raise OSError(err) - @logit(logger) def prepare_IMS(self) -> None: """Prepare the IMS data for a global snow analysis @@ -248,7 +171,7 @@ def initialize(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() - keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', + keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] @@ -272,6 +195,19 @@ def initialize(self) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") + # Read and render the GTS_LIST yaml + logger.info(f"Reading {self.task_config.GTS_LIST}") + gts_config = parse_j2yaml(self.task_config.GTS_LIST, localconf) + logger.debug(f"{self.task_config.GTS_LIST}:\n{pformat(gts_config)}") + + # Generate bufr2ioda mapping YAML files + for name in gts_config.bufr2ioda.keys(): + mapping_yaml = os.path.join(self.task_config.DATA, "obs", f"bufr_{name}_mapping.yaml") + logger.info(f"Generate BUFR2IODA YAML file: {mapping_yaml}") + temp_yaml = parse_j2yaml(gts_config.bufr2ioda[name], localconf) + save_as_yaml(temp_yaml, mapping_yaml) + logger.info(f"Wrote bufr2ioda YAML to: {mapping_yaml}") + # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ From 8d1bc8743c30b8db006d7e0731c7bd94eb1baf73 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Mon, 9 Sep 2024 20:38:56 -0400 Subject: [PATCH 2/5] Change HOMEgfs to PARMgfs. --- ush/python/pygfs/task/snow_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index ca2d4fd49b..aa51a65bf6 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -171,7 +171,7 @@ def initialize(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() - keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', + keys = ['PARMgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] From 5074d11abf0fce18b4fca2b4c31623cb0582516a Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 12 Sep 2024 07:13:51 -0400 Subject: [PATCH 3/5] Made changes to stage the static GTS mapping files. --- ush/python/pygfs/task/snow_analysis.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index aa51a65bf6..4a6c6efd8a 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -191,23 +191,15 @@ def initialize(self) -> None: logger.info("Staging ensemble backgrounds") FileHandler(self.get_ens_bkg_dict(localconf)).sync() + # stage GTS bufr2ioda mapping YAML files + logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.GTS_LIST}") + gts_mapping_list = parse_j2yaml(self.task_config.GTS_LIST, localconf) + FileHandler(gts_mapping_list).sync() + # Write out letkfoi YAML file save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") - # Read and render the GTS_LIST yaml - logger.info(f"Reading {self.task_config.GTS_LIST}") - gts_config = parse_j2yaml(self.task_config.GTS_LIST, localconf) - logger.debug(f"{self.task_config.GTS_LIST}:\n{pformat(gts_config)}") - - # Generate bufr2ioda mapping YAML files - for name in gts_config.bufr2ioda.keys(): - mapping_yaml = os.path.join(self.task_config.DATA, "obs", f"bufr_{name}_mapping.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {mapping_yaml}") - temp_yaml = parse_j2yaml(gts_config.bufr2ioda[name], localconf) - save_as_yaml(temp_yaml, mapping_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {mapping_yaml}") - # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ From f122edf47da50c78a0dee9e5dd2f00514456eebd Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 12 Sep 2024 09:35:22 -0400 Subject: [PATCH 4/5] Change the name to be more descriptive. --- parm/config/gfs/config.snowanl | 2 +- ush/python/pygfs/task/snow_analysis.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index c1a99f99ea..b1460dfa67 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -9,7 +9,7 @@ echo "BEGIN: config.snowanl" source "${EXPDIR}/config.resources" snowanl export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2" -export GTS_LIST="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" +export GTS_SNOW_STAGE_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" # Name of the JEDI executable and its yaml template export JEDIEXE="${EXECgfs}/gdas.x" diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 4a6c6efd8a..4b991d2b34 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -192,8 +192,8 @@ def initialize(self) -> None: FileHandler(self.get_ens_bkg_dict(localconf)).sync() # stage GTS bufr2ioda mapping YAML files - logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.GTS_LIST}") - gts_mapping_list = parse_j2yaml(self.task_config.GTS_LIST, localconf) + logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.GTS_SNOW_STAGE_YAML}") + gts_mapping_list = parse_j2yaml(self.task_config.GTS_SNOW_STAGE_YAML, localconf) FileHandler(gts_mapping_list).sync() # Write out letkfoi YAML file From 7de1360760011f7a169a2f74c6a644f8b6721096 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 17 Sep 2024 16:04:14 -0400 Subject: [PATCH 5/5] Update GDASApp's commit hash. --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18..7c1c181359 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 7c1c181359c2c1952bab3dc1c481bbdb361aa472