converted obsproc to obsprep

parm/soca/obsprep/obsprep_config.yaml

@@ -0,0 +1,41 @@
+observations:
+- obs space:
+    name: sss_smap
+    dmpdir subdir: SSS
+    dmpdir regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5
+    provider: SMAP
+    output file: sss_smap.ioda.nc
+- obs space:
+    name: sss_smos
+    provider: SMOS
+    dmpdir subdir: SSS
+    output file: sss_smos.ioda.nc
+    dmpdir regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc
+- obs space:
+    name: adt_rads_all
+    dmpdir subdir: ADT
+    dmpdir regex: rads_adt_??_???????.nc
+    provider: RADS
+    output file: adt_rads_all.nc4
+- obs space:
+    name: icec_amsr2_north
+    provider: AMSR2
+    dmpdir subdir: icec
+    output file: icec_amsr2_north.ioda.nc
+    dmpdir regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: icec_amsr2_south
+    provider: AMSR2
+    dmpdir subdir: icec
+    output file: icec_amsr2_south.ioda.nc
+    dmpdir regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: sst_metopb_l3u
+    provider: GHRSST
+    dmpdir subdir: 'sst'
+    output file: sst_metopb_l3u.ioda.nc
+    dmpdir regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc'
+    bounds:
+      units: C
+      min: -3.0
+      max: 50.0

scripts/exglobal_prep_ocean_obs.py

@@ -30,8 +30,13 @@
 # present in OBS_YAML_DIR
 obsConfig = YAMLFile(OBS_YAML)
 
-OBSPROC_YAML = os.getenv('OBSPROC_YAML')
-obsprocConfig = YAMLFile(OBSPROC_YAML)
+OBSPREP_YAML = os.getenv('OBSPREP_YAML')
+
+if os.path.exists(OBSPREP_YAML):
+    obsprepConfig = YAMLFile(OBSPREP_YAML)
+else:
+    print(f"CRITICAL: OBSPREP_YAML file {OBSPREP_YAML} does not exist")
+    raise FileNotFoundError
 
 filesToSave = []
 
@@ -48,41 +53,41 @@
             print("WARNING: Ill-formed observer yaml file, skipping")
             continue  # to next observer
 
-# ...look through the observations in OBSPROC_YAML...
-        for observation in obsprocConfig['observations']:
+# ...look through the observations in OBSPREP_YAML...
+        for observation in obsprepConfig['observations']:
 
-            obsprocSpace = observation['obs space']
-            obsprocSpaceName = obsprocSpace['name']
+            obsprepSpace = observation['obs space']
+            obsprepSpaceName = obsprepSpace['name']
 
 # ...for a matching name, and process the observation source
-            if obsprocSpaceName == obsSpaceName:
+            if obsprepSpaceName == obsSpaceName:
 
-                print(f"obsprocSpaceName: {obsSpaceName}")
+                print(f"obsprepSpaceName: {obsSpaceName}")
 
                 # fetch the obs files from DMPDIR to RUNDIR
-                matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace)
+                matchingFiles = prep_marine_obs.obs_fetch(obsprepSpace)
 
                 if not matchingFiles:
                     print("WARNING: No files found for obs source , skipping")
                     break  # to next observation source in OBS_YAML
 
-                obsprocSpace['input files'] = matchingFiles
-                obsprocSpace['window begin'] = windowBegin
-                obsprocSpace['window end'] = windowEnd
+                obsprepSpace['input files'] = matchingFiles
+                obsprepSpace['window begin'] = windowBegin
+                obsprepSpace['window end'] = windowEnd
                 outputFilename = f"gdas.t{cyc}z.{obsSpaceName}.{PDY}{cyc}.nc4"
-                obsprocSpace['output file'] = outputFilename
+                obsprepSpace['output file'] = outputFilename
 
-                iodaYamlFilename = obsprocSpaceName + '2ioda.yaml'
-                save_as_yaml(obsprocSpace, iodaYamlFilename)
+                iodaYamlFilename = obsprepSpaceName + '2ioda.yaml'
+                save_as_yaml(obsprepSpace, iodaYamlFilename)
 
                 subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)
 
-                filesToSave.append([obsprocSpace['output file'],
-                                    os.path.join(COMOUT_OBS, obsprocSpace['output file'])])
+                filesToSave.append([obsprepSpace['output file'],
+                                    os.path.join(COMOUT_OBS, obsprepSpace['output file'])])
                 filesToSave.append([iodaYamlFilename,
                                     os.path.join(COMOUT_OBS, iodaYamlFilename)])
 except TypeError:
-    print("CRITICAL: Ill-formed OBS_YAML file, exiting")
+    print("CRITICAL: Ill-formed OBS_YAML or OBSPREP_YAML file, exiting")
     raise
 
 if not os.path.exists(COMOUT_OBS):

test/soca/gw/CMakeLists.txt

@@ -33,12 +33,12 @@ add_test(NAME test_gdasapp_soca_run_clean
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch)
 
 # Create scratch for obs prep task
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep)
 
-# Set up obsproc, based on test_gdasapp_util_prepdata
-add_test(NAME test_gdasapp_soca_setup_obsproc
-         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+# Set up obsprep, based on test_gdasapp_util_prepdata
+add_test(NAME test_gdasapp_soca_setup_obsprep
+         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsprep.sh ${PROJECT_SOURCE_DIR}
+         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep)
 
 # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
 set(jjob_list "JGLOBAL_PREP_OCEAN_OBS"

test/soca/gw/run_jjobs.yaml.test

@@ -55,8 +55,8 @@ setup_expt config:
     NICAS_GRID_SIZE: 150
   prepoceanobs:
     SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
-    OBSPROC_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml
-    DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc
+    OBSPREP_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
+    DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsprep
 
 job options:
   account: da-cpu

test/soca/gw/setup_obsproc.sh → test/soca/gw/setup_obsprep.sh

@@ -3,7 +3,7 @@ set -ex
 
 project_source_dir=$1
 
-# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command
+# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep, set in ctest command
 test_dmpdir="gdas.20180415/12"
 
 rm -rf ${test_dmpdir}

ush/soca/prep_marine_obs.py

@@ -63,10 +63,10 @@
 }
 
 
-def obs_fetch(obsprocSpace):
+def obs_fetch(obsprepSpace):
 
-    subDir = obsprocSpace['obsproc subdir']
-    filepattern = obsprocSpace['obsproc regex']
+    subDir = obsprepSpace['dmpdir subdir']
+    filepattern = obsprepSpace['dmpdir regex']
 
     dataDir = os.path.join(cycDir, subDir)
     # TODO: check the existence of this