Merge branch 'develop' into feature/madis_da2

parm/aero/obs/config/viirs_n20_aod.yaml

@@ -24,3 +24,19 @@ obs operator:
     AerosolOption: aerosols_gocart_default
 obs error:
   covariance model: diagonal
+obs filters:
+  - filter: PreQC
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0

parm/aero/obs/config/viirs_npp_aod.yaml

@@ -26,4 +26,17 @@ obs error:
   covariance model: diagonal
 obs filters:
   - filter: PreQC
-    maxvalue: 0
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0

parm/atm/obs/testing/satwind.yaml

@@ -989,7 +989,8 @@ obs post filters:
 linear obs operator:
   name: Identity
 
-passedBenchmark: 1025814 # 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+#passedBenchmark: 1025814# 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 512907 GSI/UFO agreements, and:
                          # 16 GOES AMVs (6 u-obs, 6 v-obs) that are rejected by the SatWindsErrnormCheck in UFO
                          # but are retained in GSI's equivalent experr_norm check. All 6 of these disagreements
                          # have error norm values at almost exactly 0.9, indicating a float precision/handling

parm/atm/obs/testing/satwind_noqc.yaml

@@ -0,0 +1,18 @@
+obs space:
+  name: satwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_diag_${CDATE}.nc4
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV satwind_geoval_${CDATE}.nc4
+obs operator:
+  name: VertInterp
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5

parm/atm/obs/testing/scatwind.yaml

@@ -60,8 +60,8 @@ obs post filters:
     where:
     - variable: ObsType/windEastward
       is_in: 290
-    - variable: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1. 
+    - variable: GeoVaLs/water_area_fraction
+      maxvalue: 0.001 
     action:
       name: reject
   # Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s
@@ -144,12 +144,12 @@ obs post filters:
           variable: windNorthward
           inflation factor: 4.0
   # All scatwinds subject to a gross error check. This is contained within
-  # the SatWindsSPDBCheck, although it is not exclusive to satwinds.
+  # the WindsSPDBCheck, although it is not exclusive to satwinds.
   - filter: Background Check
     filter variables:
     - name: windEastward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -163,7 +163,7 @@ obs post filters:
     filter variables:
     - name: windNorthward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -175,7 +175,7 @@ obs post filters:
   # The last error inflation check is for duplicate observations. This one needs
   # to come last, because we don't want to inflate errors for duplication if one
   # of the duplicates should be rejected.
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windEastward
     action:
@@ -185,7 +185,7 @@ obs post filters:
         options:
           use_air_pressure: true
           variable: windEastward
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windNorthward
     action:
@@ -229,7 +229,8 @@ obs post filters:
 #      inflation factor: 1.25
 linear obs operator:
   name: Identity
-passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+#passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+passedBenchmark: 52109 # 2 variables (u,v), u=26093 passing, v=26016 passing
                        # GSI rejects both u- and v-component in first-guess check, UFO does not, but when UFO rej is reconciled btwn u and v these match GSI rej
                        #   u: 207 obs pass with corresponding v being rejected, 25880 obs in UFO/GSI agreement
                        #   v: 130 obs pass with corresponding u being rejected, 25880 obs in UFO/GSI agreement

parm/atm/obs/testing/scatwind_noqc.yaml

@@ -0,0 +1,21 @@
+obs space:
+  name: scatwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_diag_${CDATE}.nc4 
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV scatwind_geoval_${CDATE}.nc4
+vector ref: GsiHofXBc
+tolerance: 0.01
+obs operator:
+  name: VertInterp
+  apply near surface wind scaling: true
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5

parm/ioda/bufr2ioda/bufr2ioda_satwind_scat.json

@@ -0,0 +1,19 @@
+{
+  "data_format"      : "bufr_d",
+  "data_type"        : "ascatw",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "subsets"          : [ "NC012122" ],
+  "data_description" : "Level-2 ocean surface wind vector retrievals and derived wind components from ASCAT on MetOp satellites at 50-km sampling resolution",
+  "data_provider"    : "U.S. NOAA/NDESDIS",
+  "sensor_info"      : { "sensor_name": "ASCAT",  "sensor_full_name": "Advanced Scatterometer",  "sensor_id": 0 },
+  "satellite_info"   : [
+                          { "satellite_name": "MetOp-A", "satellite_full_name": "Meteorological Operational Satellite - A", "satellite_id": 4 },
+                          { "satellite_name": "MetOp-B", "satellite_full_name": "Meteorological Operational Satellite - B", "satellite_id": 3 },
+                          { "satellite_name": "MEtOp-C", "satellite_full_name": "Meteorological Operational Satellite - C", "satellite_id": 5 }
+                       ]
+}
+
+

ush/genYAML

@@ -7,7 +7,8 @@ import datetime as dt
 import logging
 import os
 import re
-from wxflow, import parse_j2yaml, cast_strdict_as_dtypedict, save_as_yaml
+from wxflow import parse_j2yaml, cast_strdict_as_dtypedict, save_as_yaml
+from wxflow import add_to_datetime, to_timedelta
 
 
 def genYAML(input_yaml, output_yaml, config):
@@ -24,4 +25,9 @@ if __name__ == "__main__":
     # Take configuration from environment and cast it as python dictionary
     config = cast_strdict_as_dtypedict(os.environ)
 
+    # need to get window begin from pdy, cyc, assim_freq
+    current_cycle = add_to_datetime(config['PDY'], to_timedelta(f"{config.cyc}H"))
+    config['WINDOW_BEGIN'] = add_to_datetime(current_cycle, to_timedelta(f"-{int(config.assim_freq/2)}H"))
+    config['WINDOW_END'] = add_to_datetime(config['WINDOW_BEGIN'], to_timedelta(f"{config.assim_freq}H"))
+
     genYAML(args.input, args.output, config)