From 4fdf179b4d98c1bf71096db5a8dd52779b7b584d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 17 Jan 2024 20:12:54 +0000 Subject: [PATCH] converted obsproc to obsprep --- parm/soca/obsprep/obsprep_config.yaml | 41 +++++++++++++++++++ parm/soca/obsproc/obsproc_config.yaml | 41 ------------------- scripts/exglobal_prep_ocean_obs.py | 41 +++++++++++-------- test/soca/gw/CMakeLists.txt | 10 ++--- test/soca/gw/run_jjobs.yaml.test | 4 +- .../gw/{setup_obsproc.sh => setup_obsprep.sh} | 2 +- ush/soca/prep_marine_obs.py | 6 +-- 7 files changed, 75 insertions(+), 70 deletions(-) create mode 100644 parm/soca/obsprep/obsprep_config.yaml delete mode 100644 parm/soca/obsproc/obsproc_config.yaml rename test/soca/gw/{setup_obsproc.sh => setup_obsprep.sh} (90%) diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml new file mode 100644 index 000000000..1be4c45d3 --- /dev/null +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -0,0 +1,41 @@ +observations: +- obs space: + name: sss_smap + dmpdir subdir: SSS + dmpdir regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5 + provider: SMAP + output file: sss_smap.ioda.nc +- obs space: + name: sss_smos + provider: SMOS + dmpdir subdir: SSS + output file: sss_smos.ioda.nc + dmpdir regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc +- obs space: + name: adt_rads_all + dmpdir subdir: ADT + dmpdir regex: rads_adt_??_???????.nc + provider: RADS + output file: adt_rads_all.nc4 +- obs space: + name: icec_amsr2_north + provider: AMSR2 + dmpdir subdir: icec + output file: icec_amsr2_north.ioda.nc + dmpdir regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: icec_amsr2_south + provider: AMSR2 + dmpdir subdir: icec + output file: icec_amsr2_south.ioda.nc + dmpdir regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: sst_metopb_l3u + provider: GHRSST + dmpdir subdir: 'sst' + output file: sst_metopb_l3u.ioda.nc + dmpdir regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' + bounds: + units: C + min: -3.0 + max: 50.0 diff --git a/parm/soca/obsproc/obsproc_config.yaml b/parm/soca/obsproc/obsproc_config.yaml deleted file mode 100644 index 5102ac9a3..000000000 --- a/parm/soca/obsproc/obsproc_config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -observations: -- obs space: - name: sss_smap - obsproc subdir: SSS - obsproc regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5 - provider: SMAP - output file: sss_smap.ioda.nc -- obs space: - name: sss_smos - provider: SMOS - obsproc subdir: SSS - output file: sss_smos.ioda.nc - obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc -- obs space: - name: adt_rads_all - obsproc subdir: ADT - obsproc regex: rads_adt_??_???????.nc - provider: RADS - output file: adt_rads_all.nc4 -- obs space: - name: icec_amsr2_north - provider: AMSR2 - obsproc subdir: icec - output file: icec_amsr2_north.ioda.nc - obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc -- obs space: - name: icec_amsr2_south - provider: AMSR2 - obsproc subdir: icec - output file: icec_amsr2_south.ioda.nc - obsproc regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc -- obs space: - name: sst_metopb_l3u - provider: GHRSST - obsproc subdir: 'sst' - output file: sst_metopb_l3u.ioda.nc - obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' - bounds: - units: C - min: -3.0 - max: 50.0 diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 437f18234..a8ab59d6b 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -30,8 +30,13 @@ # present in OBS_YAML_DIR obsConfig = YAMLFile(OBS_YAML) -OBSPROC_YAML = os.getenv('OBSPROC_YAML') -obsprocConfig = YAMLFile(OBSPROC_YAML) +OBSPREP_YAML = os.getenv('OBSPREP_YAML') + +if os.path.exists(OBSPREP_YAML): + obsprepConfig = YAMLFile(OBSPREP_YAML) +else: + print(f"CRITICAL: OBSPREP_YAML file {OBSPREP_YAML} does not exist") + raise FileNotFoundError filesToSave = [] @@ -48,41 +53,41 @@ print("WARNING: Ill-formed observer yaml file, skipping") continue # to next observer -# ...look through the observations in OBSPROC_YAML... - for observation in obsprocConfig['observations']: +# ...look through the observations in OBSPREP_YAML... + for observation in obsprepConfig['observations']: - obsprocSpace = observation['obs space'] - obsprocSpaceName = obsprocSpace['name'] + obsprepSpace = observation['obs space'] + obsprepSpaceName = obsprepSpace['name'] # ...for a matching name, and process the observation source - if obsprocSpaceName == obsSpaceName: + if obsprepSpaceName == obsSpaceName: - print(f"obsprocSpaceName: {obsSpaceName}") + print(f"obsprepSpaceName: {obsSpaceName}") # fetch the obs files from DMPDIR to RUNDIR - matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace) + matchingFiles = prep_marine_obs.obs_fetch(obsprepSpace) if not matchingFiles: print("WARNING: No files found for obs source , skipping") break # to next observation source in OBS_YAML - obsprocSpace['input files'] = matchingFiles - obsprocSpace['window begin'] = windowBegin - obsprocSpace['window end'] = windowEnd + obsprepSpace['input files'] = matchingFiles + obsprepSpace['window begin'] = windowBegin + obsprepSpace['window end'] = windowEnd outputFilename = f"gdas.t{cyc}z.{obsSpaceName}.{PDY}{cyc}.nc4" - obsprocSpace['output file'] = outputFilename + obsprepSpace['output file'] = outputFilename - iodaYamlFilename = obsprocSpaceName + '2ioda.yaml' - save_as_yaml(obsprocSpace, iodaYamlFilename) + iodaYamlFilename = obsprepSpaceName + '2ioda.yaml' + save_as_yaml(obsprepSpace, iodaYamlFilename) subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True) - filesToSave.append([obsprocSpace['output file'], - os.path.join(COMOUT_OBS, obsprocSpace['output file'])]) + filesToSave.append([obsprepSpace['output file'], + os.path.join(COMOUT_OBS, obsprepSpace['output file'])]) filesToSave.append([iodaYamlFilename, os.path.join(COMOUT_OBS, iodaYamlFilename)]) except TypeError: - print("CRITICAL: Ill-formed OBS_YAML file, exiting") + print("CRITICAL: Ill-formed OBS_YAML or OBSPREP_YAML file, exiting") raise if not os.path.exists(COMOUT_OBS): diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index bd6616744..25d290571 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -33,12 +33,12 @@ add_test(NAME test_gdasapp_soca_run_clean file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch) # Create scratch for obs prep task -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep) -# Set up obsproc, based on test_gdasapp_util_prepdata -add_test(NAME test_gdasapp_soca_setup_obsproc - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) +# Set up obsprep, based on test_gdasapp_util_prepdata +add_test(NAME test_gdasapp_soca_setup_obsprep + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsprep.sh ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep) # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_* set(jjob_list "JGLOBAL_PREP_OCEAN_OBS" diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index c4f4e12f3..dbd2fcd2f 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -55,8 +55,8 @@ setup_expt config: NICAS_GRID_SIZE: 150 prepoceanobs: SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml - OBSPROC_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml - DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc + OBSPREP_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml + DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsprep job options: account: da-cpu diff --git a/test/soca/gw/setup_obsproc.sh b/test/soca/gw/setup_obsprep.sh similarity index 90% rename from test/soca/gw/setup_obsproc.sh rename to test/soca/gw/setup_obsprep.sh index 9aa7f0583..f64bc7a54 100755 --- a/test/soca/gw/setup_obsproc.sh +++ b/test/soca/gw/setup_obsprep.sh @@ -3,7 +3,7 @@ set -ex project_source_dir=$1 -# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command +# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep, set in ctest command test_dmpdir="gdas.20180415/12" rm -rf ${test_dmpdir} diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py index 6bae569fd..5f1ec429f 100755 --- a/ush/soca/prep_marine_obs.py +++ b/ush/soca/prep_marine_obs.py @@ -63,10 +63,10 @@ } -def obs_fetch(obsprocSpace): +def obs_fetch(obsprepSpace): - subDir = obsprocSpace['obsproc subdir'] - filepattern = obsprocSpace['obsproc regex'] + subDir = obsprepSpace['dmpdir subdir'] + filepattern = obsprepSpace['dmpdir regex'] dataDir = os.path.join(cycDir, subDir) # TODO: check the existence of this