Variables for IODA format added after cleanup of the ioda converter

NOAA-EMC · Sep 21, 2023 · 4801fdd · 4801fdd
2 parents e197ffc + 1f2fc4e
commit 4801fdd
Show file tree

Hide file tree

Showing 33 changed files with 888 additions and 189 deletions.
diff --git a/parm/aero/obs/config/viirs_n20_aod.yaml b/parm/aero/obs/config/viirs_n20_aod.yaml
@@ -24,3 +24,19 @@ obs operator:
     AerosolOption: aerosols_gocart_default
 obs error:
   covariance model: diagonal
+obs filters:
+  - filter: PreQC
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0
diff --git a/parm/aero/obs/config/viirs_npp_aod.yaml b/parm/aero/obs/config/viirs_npp_aod.yaml
@@ -26,4 +26,17 @@ obs error:
   covariance model: diagonal
 obs filters:
   - filter: PreQC
-    maxvalue: 0
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0
diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_goes-16.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-16.${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-16.{{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [windEastward, windNorthward]

diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_goes-17.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-17.${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-17.{{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [windEastward, windNorthward]

diff --git a/parm/atm/obs/testing/satwind.yaml b/parm/atm/obs/testing/satwind.yaml
@@ -989,7 +989,8 @@ obs post filters:
 linear obs operator:
   name: Identity
 
-passedBenchmark: 1025814 # 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+#passedBenchmark: 1025814# 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 512907 GSI/UFO agreements, and:
                          # 16 GOES AMVs (6 u-obs, 6 v-obs) that are rejected by the SatWindsErrnormCheck in UFO
                          # but are retained in GSI's equivalent experr_norm check. All 6 of these disagreements
                          # have error norm values at almost exactly 0.9, indicating a float precision/handling

diff --git a/parm/atm/obs/testing/satwind_noqc.yaml b/parm/atm/obs/testing/satwind_noqc.yaml
@@ -0,0 +1,18 @@
+obs space:
+  name: satwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_diag_${CDATE}.nc4
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV satwind_geoval_${CDATE}.nc4
+obs operator:
+  name: VertInterp
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5
diff --git a/parm/atm/obs/testing/scatwind.yaml b/parm/atm/obs/testing/scatwind.yaml
@@ -60,8 +60,8 @@ obs post filters:
     where:
     - variable: ObsType/windEastward
       is_in: 290
-    - variable: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1. 
+    - variable: GeoVaLs/water_area_fraction
+      maxvalue: 0.001 
     action:
       name: reject
   # Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s
@@ -144,12 +144,12 @@ obs post filters:
           variable: windNorthward
           inflation factor: 4.0
   # All scatwinds subject to a gross error check. This is contained within
-  # the SatWindsSPDBCheck, although it is not exclusive to satwinds.
+  # the WindsSPDBCheck, although it is not exclusive to satwinds.
   - filter: Background Check
     filter variables:
     - name: windEastward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -163,7 +163,7 @@ obs post filters:
     filter variables:
     - name: windNorthward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -175,7 +175,7 @@ obs post filters:
   # The last error inflation check is for duplicate observations. This one needs
   # to come last, because we don't want to inflate errors for duplication if one
   # of the duplicates should be rejected.
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windEastward
     action:
@@ -185,7 +185,7 @@ obs post filters:
         options:
           use_air_pressure: true
           variable: windEastward
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windNorthward
     action:
@@ -229,7 +229,8 @@ obs post filters:
 #      inflation factor: 1.25
 linear obs operator:
   name: Identity
-passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+#passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+passedBenchmark: 52109 # 2 variables (u,v), u=26093 passing, v=26016 passing
                        # GSI rejects both u- and v-component in first-guess check, UFO does not, but when UFO rej is reconciled btwn u and v these match GSI rej
                        #   u: 207 obs pass with corresponding v being rejected, 25880 obs in UFO/GSI agreement
                        #   v: 130 obs pass with corresponding u being rejected, 25880 obs in UFO/GSI agreement

diff --git a/parm/atm/obs/testing/scatwind_noqc.yaml b/parm/atm/obs/testing/scatwind_noqc.yaml
@@ -0,0 +1,21 @@
+obs space:
+  name: scatwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_diag_${CDATE}.nc4 
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV scatwind_geoval_${CDATE}.nc4
+vector ref: GsiHofXBc
+tolerance: 0.01
+obs operator:
+  name: VertInterp
+  apply near surface wind scaling: true
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5
diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_scat.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_scat.json
@@ -0,0 +1,19 @@
+{
+  "data_format"      : "bufr_d",
+  "data_type"        : "ascatw",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "subsets"          : [ "NC012122" ],
+  "data_description" : "Level-2 ocean surface wind vector retrievals and derived wind components from ASCAT on MetOp satellites at 50-km sampling resolution",
+  "data_provider"    : "U.S. NOAA/NDESDIS",
+  "sensor_info"      : { "sensor_name": "ASCAT",  "sensor_full_name": "Advanced Scatterometer",  "sensor_id": 0 },
+  "satellite_info"   : [
+                          { "satellite_name": "MetOp-A", "satellite_full_name": "Meteorological Operational Satellite - A", "satellite_id": 4 },
+                          { "satellite_name": "MetOp-B", "satellite_full_name": "Meteorological Operational Satellite - B", "satellite_id": 3 },
+                          { "satellite_name": "MEtOp-C", "satellite_full_name": "Meteorological Operational Satellite - C", "satellite_id": 5 }
+                       ]
+}
+
+
diff --git a/parm/land/obs/config/snocvr_snow.yaml b/parm/land/obs/config/snocvr_snow.yaml
@@ -0,0 +1,102 @@
+obs space:
+  name: snocvr_snow
+  distribution:
+    name: Halo
+    halo size: 250e3
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: '{{ DATA }}/obs/{{ OPREFIX }}snocvr_snow.nc4'
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: '{{ DATA }}/diags/diag_snocvr_snow.nc4'
+  simulated variables: [totalSnowDepth]
+obs operator:
+  name: Composite
+  components:
+  - name: Identity
+  - name: BackgroundErrorIdentity
+obs error:
+  covariance model: diagonal
+obs localizations:
+  - localization method: Horizontal SOAR
+    lengthscale: 250e3
+    soar horizontal decay: 0.000021
+    max nobs: 50
+  - localization method: Vertical Brasnett
+    vertical lengthscale: 700
+obs filters:
+  - filter: Bounds Check
+    filter variables:
+    - name: totalSnowDepth
+    minvalue: 0.0
+    maxvalue: 2000.0
+    action:
+      name: reject
+  - filter: Domain Check
+    where:
+    - variable:
+        name: MetaData/height
+      minvalue: -999.0
+  - filter: Domain Check # land only
+    where:
+    - variable:
+        name: GeoVaLs/slmsk
+      minvalue: 0.5
+      maxvalue: 1.5
+  - filter: RejectList  # no land-ice
+    where:
+    - variable:
+        name: GeoVaLs/vtype
+      minvalue: 14.5
+      maxvalue: 15.5
+  - filter: Temporal Thinning
+    min_spacing: '{{ LAND_WINDOW_LENGTH }}'
+    seed_time: '{{ current_cycle | to_isotime }}'
+    category_variable:
+      name: MetaData/stationIdentification
+  - filter: Background Check # gross error check
+    filter variables:
+    - name: totalSnowDepth
+    threshold: 6.25
+    action:
+      name: reject
+  - filter: Met Office Buddy Check
+    filter variables:
+    - name: totalSnowDepth
+      rejection_threshold: 0.5
+      traced_boxes: # trace all observations
+        min_latitude: -90
+        max_latitude:  90
+        min_longitude: -180
+        max_longitude:  180
+      search_radius: 150 # km
+      station_id_variable:
+        name: MetaData/stationIdentification
+      num_zonal_bands: 24
+      sort_by_pressure: false
+      max_total_num_buddies: 15
+      max_num_buddies_from_single_band: 10
+      max_num_buddies_with_same_station_id: 5
+      use_legacy_buddy_collector: false
+      horizontal_correlation_scale: { "-90": 150, "90": 150 }
+      temporal_correlation_scale: PT6H
+      damping_factor_1: 1.0
+      damping_factor_2: 1.0
+      background_error_group: BkgError
+  - filter: Perform Action
+    filter variables:
+    - name: totalSnowDepth
+    action:
+      name: assign error
+      error parameter: 40.0
+  - filter: Variable Assignment
+    assignments:
+    - name: GrossErrorProbability/totalSnowDepth
+      type: float
+      value: 0.02
+    - name: BkgError/totalSnowDepth_background_error
+      type: float
+      value: 30.0
+
diff --git a/parm/land/obs/lists/gdas_land_gts_only.yaml b/parm/land/obs/lists/gdas_land_gts_only.yaml
@@ -0,0 +1,3 @@
+observers:
+- !INC ${OBS_YAML_DIR}/adpsfc_snow.yaml
+- !INC ${OBS_YAML_DIR}/snocvr_snow.yaml
diff --git a/parm/land/obs/lists/gdas_land_prototype.yaml b/parm/land/obs/lists/gdas_land_prototype.yaml
@@ -1,3 +1,4 @@
 observers:
 - !INC ${OBS_YAML_DIR}/adpsfc_snow.yaml
+- !INC ${OBS_YAML_DIR}/snocvr_snow.yaml
 - !INC ${OBS_YAML_DIR}/ims_snow.yaml
diff --git a/parm/land/prep/prep_gts.yaml b/parm/land/prep/prep_gts.yaml
@@ -2,8 +2,12 @@ gtsbufr:
   mkdir:
     - $(DATA)/obs
   copy:
-    - [$(COM_OBS)/$(OPREFIX)adpsfc.tm00.bufr_d, $(DATA)/obs/]
+    - [{{ COM_OBS }}/{{ OPREFIX }}adpsfc.tm00.bufr_d, {{ DATA }}/obs/]
+    - [{{ COM_OBS }}/{{ OPREFIX }}snocvr.tm00.bufr_d, {{ DATA }}/obs/]
 gtsioda:
   copy:
-    - [$(DATA)/$(OPREFIX)adpsfc_snow.nc4, $(COM_OBS)/$(OPREFIX)adpsfc_snow.nc4]
-
+    - [{{ DATA }}/{{ OPREFIX }}adpsfc_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}adpsfc_snow.nc4]
+    - [{{ DATA }}/{{ OPREFIX }}snocvr_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}snocvr_snow.nc4]
+bufr2ioda:
+     adpsfc: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml
+     snocvr: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_snocvr_snow.yaml
diff --git a/parm/soca/berror/saber_blocks.yaml b/parm/soca/berror/saber_blocks.yaml
@@ -32,21 +32,23 @@ components:
         ice_filename: 'ice.bkgerr_stddev.incr.{{ATM_WINDOW_BEGIN}}.nc'
         date: '{{ATM_WINDOW_MIDDLE}}'
         t_min: 0.1
-        t_max: 10.0
+        t_max: 5.0
         s_min: 0.1
-        s_max: 10.0
+        s_max: 1.0
         ssh_min: 0.0 # std ssh=0 => ssh balance applied as
-        ssh_max: 0.0 #              strong constraint
+        ssh_max: 1.0 #              strong constraint
         cicen_min: 0.1
         cicen_max: 0.5
         hicen_min: 0.0
         hicen_max: 0.0
         standard deviation: true
 
       - linear variable change name: BalanceSOCA
+        ksshts:
+          nlayers: 10
 
   weight:
-    value: 1.0
+    value: 0.25
 - covariance:
     covariance model: ensemble
     members from template:

diff --git a/parm/soca/berror/soca_ensweights.yaml b/parm/soca/berror/soca_ensweights.yaml
@@ -18,7 +18,7 @@ background:
 weights:
   # Need to provide weights^2 when reading from file
   ice: 0.0025   #  5% of original variance
-  ocean: 0.01   # 10%      "       "
+  ocean: 0.0625   # 25%      "       "
 
 output:
   datadir: ./

diff --git a/scripts/exgdas_global_marine_analysis_vrfy.py b/scripts/exgdas_global_marine_analysis_vrfy.py
@@ -48,7 +48,7 @@
 config = plotConfig(grid_file=grid_file,
                     data_file=data_file,
                     variables_horiz={'ave_ssh': [-1, 1]},
-                    colormap='gist_ncar',
+                    colormap='seismic',
                     comout=os.path.join(comout, 'vrfy', 'recentering_error'))
 recErrPlotter = statePlotter(config)
 recErrPlotter.plot()

diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test
@@ -11,6 +11,9 @@ gw environement:
     HOMEgfs: @HOMEgfs@
     COMPONENT: ocean
     RUN: gdas
+    CDUMP: gdas
+    COMROOT: dummy_COMROOT
+    jobid: dummy_jobid
 
   cycle info:
     PDY: 20180415