Updates to run gw ctests using feature/submodule

parm/land/obs/config/adpsfc_snow.yaml

@@ -26,7 +26,22 @@ obs localizations:
     max nobs: 50
   - localization method: Vertical Brasnett
     vertical lengthscale: 700
-obs filters:
+obs pre filters:
+  - filter: Perform Action
+    filter variables:
+    - name: totalSnowDepth
+    action:
+      name: assign error
+      error parameter: 40.0
+  - filter: Variable Assignment
+    assignments:
+    - name: GrossErrorProbability/totalSnowDepth
+      type: float
+      value: 0.02
+    - name: BkgError/totalSnowDepth_background_error
+      type: float
+      value: 30.0
+obs prior filters:
   - filter: Bounds Check
     filter variables:
     - name: totalSnowDepth
@@ -37,7 +52,7 @@ obs filters:
   - filter: Domain Check
     where:
     - variable:
-        name: MetaData/height
+        name: MetaData/stationElevation
       minvalue: -999.0
   - filter: Domain Check # land only
     where:
@@ -57,12 +72,18 @@ obs filters:
         name: MetaData/stationIdentification
       is_in: [71120,71397,71621,71727,71816]
       size where true: 5
+obs post filters:
   - filter: Background Check # gross error check
     filter variables:
     - name: totalSnowDepth
     threshold: 6.25
     action:
       name: reject
+  - filter: Temporal Thinning
+    min_spacing: '{{ LAND_WINDOW_LENGTH }}'
+    seed_time: '{{ current_cycle | to_isotime }}'
+    category_variable:
+      name: MetaData/stationIdentification
   - filter: Met Office Buddy Check
     filter variables:
     - name: totalSnowDepth
@@ -86,18 +107,4 @@ obs filters:
       damping_factor_1: 1.0
       damping_factor_2: 1.0
       background_error_group: BkgError
-  - filter: Perform Action
-    filter variables:
-    - name: totalSnowDepth
-    action:
-      name: assign error
-      error parameter: 40.0
-  - filter: Variable Assignment
-    assignments:
-    - name: GrossErrorProbability/totalSnowDepth
-      type: float
-      value: 0.02
-    - name: BkgError/totalSnowDepth_background_error
-      type: float
-      value: 30.0
 

parm/land/obs/config/snocvr_snow.yaml

@@ -26,7 +26,22 @@ obs localizations:
     max nobs: 50
   - localization method: Vertical Brasnett
     vertical lengthscale: 700
-obs filters:
+obs pre filters:
+  - filter: Perform Action
+    filter variables:
+    - name: totalSnowDepth
+    action:
+      name: assign error
+      error parameter: 40.0
+  - filter: Variable Assignment
+    assignments:
+    - name: GrossErrorProbability/totalSnowDepth
+      type: float
+      value: 0.02
+    - name: BkgError/totalSnowDepth_background_error
+      type: float
+      value: 30.0
+obs prior filters:
   - filter: Bounds Check
     filter variables:
     - name: totalSnowDepth
@@ -37,7 +52,7 @@ obs filters:
   - filter: Domain Check
     where:
     - variable:
-        name: MetaData/height
+        name: MetaData/stationElevation
       minvalue: -999.0
   - filter: Domain Check # land only
     where:
@@ -51,17 +66,18 @@ obs filters:
         name: GeoVaLs/vtype
       minvalue: 14.5
       maxvalue: 15.5
-  - filter: Temporal Thinning
-    min_spacing: '{{ LAND_WINDOW_LENGTH }}'
-    seed_time: '{{ current_cycle | to_isotime }}'
-    category_variable:
-      name: MetaData/stationIdentification
+obs post filters:
   - filter: Background Check # gross error check
     filter variables:
     - name: totalSnowDepth
     threshold: 6.25
     action:
       name: reject
+  - filter: Temporal Thinning
+    min_spacing: '{{ LAND_WINDOW_LENGTH }}'
+    seed_time: '{{ current_cycle | to_isotime }}'
+    category_variable:
+      name: MetaData/stationIdentification
   - filter: Met Office Buddy Check
     filter variables:
     - name: totalSnowDepth
@@ -85,18 +101,3 @@ obs filters:
       damping_factor_1: 1.0
       damping_factor_2: 1.0
       background_error_group: BkgError
-  - filter: Perform Action
-    filter variables:
-    - name: totalSnowDepth
-    action:
-      name: assign error
-      error parameter: 40.0
-  - filter: Variable Assignment
-    assignments:
-    - name: GrossErrorProbability/totalSnowDepth
-      type: float
-      value: 0.02
-    - name: BkgError/totalSnowDepth_background_error
-      type: float
-      value: 30.0
-

parm/land/prep/prep_gts.yaml

@@ -3,11 +3,8 @@ gtsbufr:
     - $(DATA)/obs
   copy:
     - [{{ COM_OBS }}/{{ OPREFIX }}adpsfc.tm00.bufr_d, {{ DATA }}/obs/]
-    - [{{ COM_OBS }}/{{ OPREFIX }}snocvr.tm00.bufr_d, {{ DATA }}/obs/]
 gtsioda:
   copy:
     - [{{ DATA }}/{{ OPREFIX }}adpsfc_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}adpsfc_snow.nc4]
-    - [{{ DATA }}/{{ OPREFIX }}snocvr_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}snocvr_snow.nc4]
 bufr2ioda:
      adpsfc: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml
-     snocvr: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_snocvr_snow.yaml

parm/land/prep/prep_ims.yaml

@@ -6,5 +6,5 @@ calcfims:
     - [$(COM_OBS)/$(OPREFIX)IMS4km_to_FV3_mapping.$(CASE)_oro_data.nc, $(DATA)/obs/IMS4km_to_FV3_mapping.$(CASE)_oro_data.nc]
 ims2ioda:
   copy:
-    - [$(DATA)/ims_snow_{{ current_cycle | to_YMDH }}.nc4, $(COM_OBS)/$(OPREFIX)ims_snow.nc4]
+    - [$(DATA)/ims_snow_{{ current_cycle | to_YMDH }}.nc4, $(DATA)/obs/$(OPREFIX)ims_snow.nc4]
 

scripts/exgdas_global_marine_analysis_prep.py

@@ -85,9 +85,10 @@ def cice_hist2fms(input_filename, output_filename):
     """
     Simple reformatting utility to allow soca/fms to read CICE's history
     """
+    input_filename_real = os.path.realpath(input_filename)
 
     # open the CICE history file
-    ds = xr.open_dataset(input_filename)
+    ds = xr.open_dataset(input_filename_real)
 
     if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables:
         logging.info(f"*** Already reformatted, skipping.")
@@ -100,7 +101,8 @@ def cice_hist2fms(input_filename, output_filename):
     ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'})
 
     # Save the new netCDF file
-    ds.to_netcdf(output_filename, mode='w')
+    output_filename_real = os.path.realpath(output_filename)
+    ds.to_netcdf(output_filename_real, mode='w')
 
 
 def test_hist_date(histfile, ref_date):
@@ -280,7 +282,8 @@ def find_clim_ens(input_date):
 
 # concatenate altimeters into one obs space
 # TODO (SAMG)temporary, move this into the obs procecing eventually
-adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt"
+com_obs = os.path.realpath(f"{os.getenv('COM_OBS')}")
+adt_obs = f"{com_obs}/{RUN}.t{cyc}z.adt"
 ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True)
 
 # get the list of observations
@@ -293,7 +296,7 @@ def find_clim_ens(input_date):
 for obs_file in obs_files:
     logging.info(f"******* {obs_file}")
     obs_src = os.path.join(os.getenv('COM_OBS'), obs_file)
-    obs_dst = os.path.join(os.path.abspath(obs_in), obs_file)
+    obs_dst = os.path.join(os.path.realpath(obs_in), obs_file)
     if os.path.exists(obs_src):
         logging.info(f"******* fetching {obs_file}")
         obs_list.append([obs_src, obs_dst])
@@ -319,20 +322,21 @@ def find_clim_ens(input_date):
     for mem in range(1, nmem_ens+1):
         for domain in ['ocn', 'ice']:
             # TODO(Guillaume): make use and define ensemble COM in the j-job
+            ensroot = os.getenv('COM_OCEAN_HISTORY_PREV')
             ensdir = os.path.join(os.getenv('COM_OCEAN_HISTORY_PREV'), '..', '..', '..', '..', '..',
                                   f'enkf{RUN}.{PDY}', f'{gcyc}', f'mem{str(mem).zfill(3)}',
                                   'model_data', longname[domain], 'history')
-            ensdir = os.path.normpath(ensdir)
+            ensdir_real = os.path.realpath(ensdir)
             f009 = f'enkfgdas.t{gcyc}z.{domain}f009.nc'
 
-            fname_in = os.path.abspath(os.path.join(ensdir, f009))
-            fname_out = os.path.abspath(os.path.join(static_ens, domain+"."+str(mem)+".nc"))
+            fname_in = os.path.abspath(os.path.join(ensdir_real, f009))
+            fname_out = os.path.realpath(os.path.join(static_ens, domain+"."+str(mem)+".nc"))
             ens_member_list.append([fname_in, fname_out])
     FileHandler({'copy': ens_member_list}).sync()
 
     # reformat the cice history output
     for mem in range(1, nmem_ens+1):
-        cice_fname = os.path.abspath(os.path.join(static_ens, "ice."+str(mem)+".nc"))
+        cice_fname = os.path.realpath(os.path.join(static_ens, "ice."+str(mem)+".nc"))
         cice_hist2fms(cice_fname, cice_fname)
 else:
     logging.info("---------------- Stage offline ensemble members")
@@ -357,8 +361,8 @@ def find_clim_ens(input_date):
 # copy yaml for grid generation
 
 logging.info(f"---------------- generate gridgen.yaml")
-gridgen_yaml_src = os.path.abspath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml'))
-gridgen_yaml_dst = os.path.abspath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml'))
+gridgen_yaml_src = os.path.realpath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml'))
+gridgen_yaml_dst = os.path.realpath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml'))
 FileHandler({'copy': [[gridgen_yaml_src, gridgen_yaml_dst]]}).sync()
 
 ################################################################################

scripts/exgdas_global_marine_analysis_vrfy.py

@@ -26,9 +26,9 @@
 import subprocess
 from datetime import datetime, timedelta
 
-comout = os.getenv('COM_OCEAN_ANALYSIS')
-com_ice_history = os.getenv('COM_ICE_HISTORY_PREV')
-com_ocean_history = os.getenv('COM_OCEAN_HISTORY_PREV')
+comout = os.path.realpath(os.getenv('COM_OCEAN_ANALYSIS'))
+com_ice_history = os.path.realpath(os.getenv('COM_ICE_HISTORY_PREV'))
+com_ocean_history = os.path.realpath(os.getenv('COM_OCEAN_HISTORY_PREV'))
 cyc = os.getenv('cyc')
 RUN = os.getenv('RUN')
 gcyc = str((int(cyc) - 6) % 24).zfill(2)

scripts/exglobal_prep_ocean_obs.py

@@ -4,7 +4,7 @@
 from datetime import datetime, timedelta
 import logging
 import os
-import prep_marine_obs
+from soca import prep_marine_obs
 import subprocess
 from wxflow import YAMLFile, save_as_yaml, FileHandler
 

test/soca/gw/CMakeLists.txt

@@ -27,7 +27,7 @@ add_test(NAME test_gdasapp_soca_concatioda
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda)
        set_tests_properties(test_gdasapp_soca_concatioda
       PROPERTIES
-      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 
 # Clean-up
 add_test(NAME test_gdasapp_soca_run_clean
@@ -78,7 +78,7 @@ foreach(jjob ${jjob_list})
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
   set_tests_properties(${test_name}
     PROPERTIES
-    ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+    ENVIRONMENT "PYTHONPATH=${PROJECT_SOURCE_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 
 
   set(setup "--skip") # Only run the setup of the first test, if not, it will hang
@@ -89,7 +89,7 @@ endforeach()
 set(ctest_list "socahybridweights" "incr_handler" "ens_handler")
 foreach(ctest ${ctest_list})
   set(TEST ${ctest})
-  set(EXEC ${PROJECT_BINARY_DIR}/bin/gdas_${ctest}.x)
+  set(EXEC ${PROJECT_BINARY_DIR}/../bin/gdas_${ctest}.x)
   set(YAML ${PROJECT_SOURCE_DIR}/test/soca/testinput/${ctest}.yaml)
   configure_file(${PROJECT_SOURCE_DIR}/test/soca/gw/run_gdas_apps.yaml.test
                  ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_gdas_apps_${ctest}.yaml)
@@ -102,5 +102,5 @@ foreach(ctest ${ctest_list})
 
   set_tests_properties(${test_name}
       PROPERTIES
-      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 endforeach()

test/soca/gw/prep.sh

@@ -35,7 +35,7 @@ done
 for day in $(seq 1 2 9); do
     cp ${COM}/06/model_data/ocean/history/gdas.t06z.ocnf003.nc \
        ${project_binary_dir}/soca_static/bkgerr/stddev/ocn.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc
-    cp ${project_source_dir}/soca/test/Data/72x35x25/ice.bkgerror.nc \
+    cp ${project_source_dir}/sorc/soca/test/Data/72x35x25/ice.bkgerror.nc \
        ${project_binary_dir}/soca_static/bkgerr/stddev/ice.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc
 done
 

test/soca/gw/run_jjobs.yaml.test

@@ -21,12 +21,12 @@ gw environement:
     assym_freq: 6
 
   backgrounds:
-    COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/COM
+    COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/COM
 
   working directories:
-    ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/ROTDIRS
-    EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/experiments
-    STMP: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs
+    ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/ROTDIRS
+    EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/experiments
+    STMP: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs
 
   jedi:
     OOPS_TRACE: 1
@@ -46,10 +46,10 @@ setup_expt config:
     NMEM_ENS: "4"
     DOHYBVAR: "YES"
   ocnanal:
-    SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static
+    SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/soca_static
     CASE_ANL: C48
     SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml
-    COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/obs/r2d2-shared
+    COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/obs/r2d2-shared
     SOCA_NINNER: 1
     R2D2_OBS_SRC: gdasapp
     R2D2_OBS_DUMP: soca

test/soca/gw/static.sh

@@ -10,12 +10,12 @@ mkdir -p ${soca_static}/INPUT
 mkdir -p ${soca_static}/bump
 mkdir -p ${soca_static}/bkgerr/stddev
 
-lowres=${project_source_dir}/soca/test/Data
+lowres=${project_source_dir}/sorc/soca/test/Data
 
 cp -L ${lowres}/workdir/{diag_table,field_table} ${soca_static}
 cp -L ${project_source_dir}/test/soca/fix/MOM_input ${soca_static}
 cp -L ${lowres}/{fields_metadata.yml,godas_sst_bgerr.nc,rossrad.dat} ${soca_static}
 mv ${soca_static}/fields_metadata.yml ${soca_static}/fields_metadata.yaml
-cp -L ${project_source_dir}/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml
+cp -L ${project_source_dir}/sorc/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml
 cp -L ${lowres}/72x35x25/input.nml ${soca_static}/inputnml
 cp -L ${lowres}/72x35x25/INPUT/{hycom1_25.nc,ocean_mosaic.nc,grid_spec.nc,layer_coord25.nc,ocean_hgrid.nc,ocean_topog.nc} ${soca_static}/INPUT

test/testinput/bufr_adpsfc_snow.yaml

@@ -24,7 +24,7 @@ observations:
           stationIdentification:
             query: "*/RPID"
 
-          height:
+          stationElevation:
             query: "[*/SELV, */HSMSL]"
 
           # ObsValue
@@ -64,9 +64,9 @@ observations:
           units: "degree_east"
           range: [-180, 180]
 
-        - name: "MetaData/height"
+        - name: "MetaData/stationElevation"
           coordinates: "longitude latitude"
-          source: variables/height
+          source: variables/stationElevation
           longName: "Height of Station"
 
         - name: "MetaData/stationIdentification"