Merge branch 'develop' into feature/NickE_sfcshp_prepbufr

CMakeLists.txt

@@ -74,7 +74,7 @@ if(BUILD_GDASBUNDLE)
   ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop )
 
 # Core JEDI repositories
-  ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda/oops.git"  BRANCH develop )
+  ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda/oops.git"  BRANCH develop)
   ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop )
   ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop )
   ecbuild_bundle( PROJECT ioda  GIT "https://github.com/jcsda/ioda.git" BRANCH develop  )

parm/aero/obs/config/viirs_n20_aod.yaml

@@ -24,3 +24,19 @@ obs operator:
     AerosolOption: aerosols_gocart_default
 obs error:
   covariance model: diagonal
+obs filters:
+  - filter: PreQC
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0

parm/aero/obs/config/viirs_npp_aod.yaml

@@ -26,4 +26,17 @@ obs error:
   covariance model: diagonal
 obs filters:
   - filter: PreQC
-    maxvalue: 0
+    maxvalue: 1
+  - filter: Bounds Check
+    filter variables:
+    - name: aerosolOpticalDepth
+      channels: 4
+    minvalue: 0
+    action:
+      name: reject
+  - filter: Background Check
+    channels: 4
+    threshold: 3.0
+    action:
+      name: inflate error
+      inflation factor: 3.0

parm/atm/obs/config/satwind_goes-16.yaml

@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-16.${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-16.{{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [windEastward, windNorthward]

parm/atm/obs/config/satwind_goes-17.yaml

@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine:
       type: H5File
-      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-17.${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)satwind.abi_goes-17.{{ current_cycle | to_YMDH }}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_${{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc4
   io pool:
     max pool size: 1
   simulated variables: [windEastward, windNorthward]

parm/atm/obs/testing/satwind.yaml

@@ -989,7 +989,8 @@ obs post filters:
 linear obs operator:
   name: Identity
 
-passedBenchmark: 1025814 # 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+#passedBenchmark: 1025814# 2 variables (u,v), both passing 512907 obs, including 512907 GSI/UFO agreements, and:
+passedBenchmark: 1150870 # 2 variables (u,v), both passing 575435 obs, including 512907 GSI/UFO agreements, and:
                          # 16 GOES AMVs (6 u-obs, 6 v-obs) that are rejected by the SatWindsErrnormCheck in UFO
                          # but are retained in GSI's equivalent experr_norm check. All 6 of these disagreements
                          # have error norm values at almost exactly 0.9, indicating a float precision/handling

parm/atm/obs/testing/satwind_noqc.yaml

@@ -0,0 +1,18 @@
+obs space:
+  name: satwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV satwind_diag_${CDATE}.nc4
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV satwind_geoval_${CDATE}.nc4
+obs operator:
+  name: VertInterp
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5

parm/atm/obs/testing/scatwind.yaml

@@ -60,8 +60,8 @@ obs post filters:
     where:
     - variable: ObsType/windEastward
       is_in: 290
-    - variable: GeoVaLs/land_type_index_NPOESS
-      minvalue: 1. 
+    - variable: GeoVaLs/water_area_fraction
+      maxvalue: 0.001 
     action:
       name: reject
   # Reject ASCAT (Type 290) when observed component deviates from background by more than 5.0 m/s
@@ -144,12 +144,12 @@ obs post filters:
           variable: windNorthward
           inflation factor: 4.0
   # All scatwinds subject to a gross error check. This is contained within
-  # the SatWindsSPDBCheck, although it is not exclusive to satwinds.
+  # the WindsSPDBCheck, although it is not exclusive to satwinds.
   - filter: Background Check
     filter variables:
     - name: windEastward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -163,7 +163,7 @@ obs post filters:
     filter variables:
     - name: windNorthward
     function absolute threshold:
-    - name: ObsFunction/SatWindsSPDBCheck
+    - name: ObsFunction/WindsSPDBCheck
       options:
         wndtype: [  290,  291]
         cgross:  [  5.0,  5.0]
@@ -175,7 +175,7 @@ obs post filters:
   # The last error inflation check is for duplicate observations. This one needs
   # to come last, because we don't want to inflate errors for duplication if one
   # of the duplicates should be rejected.
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windEastward
     action:
@@ -185,7 +185,7 @@ obs post filters:
         options:
           use_air_pressure: true
           variable: windEastward
-  - filter: BlackList
+  - filter: RejectList
     filter variables:
     - name: windNorthward
     action:
@@ -229,7 +229,8 @@ obs post filters:
 #      inflation factor: 1.25
 linear obs operator:
   name: Identity
-passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+#passedBenchmark: 52097 # 2 variables (u,v), u=26087 passing, v=26010 passing
+passedBenchmark: 52109 # 2 variables (u,v), u=26093 passing, v=26016 passing
                        # GSI rejects both u- and v-component in first-guess check, UFO does not, but when UFO rej is reconciled btwn u and v these match GSI rej
                        #   u: 207 obs pass with corresponding v being rejected, 25880 obs in UFO/GSI agreement
                        #   v: 130 obs pass with corresponding u being rejected, 25880 obs in UFO/GSI agreement

parm/atm/obs/testing/scatwind_noqc.yaml

@@ -0,0 +1,21 @@
+obs space:
+  name: scatwind
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_obs_${CDATE}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV scatwind_diag_${CDATE}.nc4 
+  simulated variables: [windEastward, windNorthward]
+geovals:
+  filename: !ENV scatwind_geoval_${CDATE}.nc4
+vector ref: GsiHofXBc
+tolerance: 0.01
+obs operator:
+  name: VertInterp
+  apply near surface wind scaling: true
+
+vector ref: GsiHofXBc
+tolerance: 1.e-5

parm/ioda/bufr2ioda/bufr2ioda_satwind_scat.json

@@ -0,0 +1,19 @@
+{
+  "data_format"      : "bufr_d",
+  "data_type"        : "ascatw",
+  "cycle_type"       : "{{ RUN }}",
+  "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"   : "{{ DMPDIR }}",
+  "ioda_directory"   : "{{ COM_OBS }}",
+  "subsets"          : [ "NC012122" ],
+  "data_description" : "Level-2 ocean surface wind vector retrievals and derived wind components from ASCAT on MetOp satellites at 50-km sampling resolution",
+  "data_provider"    : "U.S. NOAA/NDESDIS",
+  "sensor_info"      : { "sensor_name": "ASCAT",  "sensor_full_name": "Advanced Scatterometer",  "sensor_id": 0 },
+  "satellite_info"   : [
+                          { "satellite_name": "MetOp-A", "satellite_full_name": "Meteorological Operational Satellite - A", "satellite_id": 4 },
+                          { "satellite_name": "MetOp-B", "satellite_full_name": "Meteorological Operational Satellite - B", "satellite_id": 3 },
+                          { "satellite_name": "MEtOp-C", "satellite_full_name": "Meteorological Operational Satellite - C", "satellite_id": 5 }
+                       ]
+}
+
+

parm/land/obs/config/snocvr_snow.yaml

@@ -0,0 +1,102 @@
+obs space:
+  name: snocvr_snow
+  distribution:
+    name: Halo
+    halo size: 250e3
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: '{{ DATA }}/obs/{{ OPREFIX }}snocvr_snow.nc4'
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: '{{ DATA }}/diags/diag_snocvr_snow.nc4'
+  simulated variables: [totalSnowDepth]
+obs operator:
+  name: Composite
+  components:
+  - name: Identity
+  - name: BackgroundErrorIdentity
+obs error:
+  covariance model: diagonal
+obs localizations:
+  - localization method: Horizontal SOAR
+    lengthscale: 250e3
+    soar horizontal decay: 0.000021
+    max nobs: 50
+  - localization method: Vertical Brasnett
+    vertical lengthscale: 700
+obs filters:
+  - filter: Bounds Check
+    filter variables:
+    - name: totalSnowDepth
+    minvalue: 0.0
+    maxvalue: 2000.0
+    action:
+      name: reject
+  - filter: Domain Check
+    where:
+    - variable:
+        name: MetaData/height
+      minvalue: -999.0
+  - filter: Domain Check # land only
+    where:
+    - variable:
+        name: GeoVaLs/slmsk
+      minvalue: 0.5
+      maxvalue: 1.5
+  - filter: RejectList  # no land-ice
+    where:
+    - variable:
+        name: GeoVaLs/vtype
+      minvalue: 14.5
+      maxvalue: 15.5
+  - filter: Temporal Thinning
+    min_spacing: '{{ LAND_WINDOW_LENGTH }}'
+    seed_time: '{{ current_cycle | to_isotime }}'
+    category_variable:
+      name: MetaData/stationIdentification
+  - filter: Background Check # gross error check
+    filter variables:
+    - name: totalSnowDepth
+    threshold: 6.25
+    action:
+      name: reject
+  - filter: Met Office Buddy Check
+    filter variables:
+    - name: totalSnowDepth
+      rejection_threshold: 0.5
+      traced_boxes: # trace all observations
+        min_latitude: -90
+        max_latitude:  90
+        min_longitude: -180
+        max_longitude:  180
+      search_radius: 150 # km
+      station_id_variable:
+        name: MetaData/stationIdentification
+      num_zonal_bands: 24
+      sort_by_pressure: false
+      max_total_num_buddies: 15
+      max_num_buddies_from_single_band: 10
+      max_num_buddies_with_same_station_id: 5
+      use_legacy_buddy_collector: false
+      horizontal_correlation_scale: { "-90": 150, "90": 150 }
+      temporal_correlation_scale: PT6H
+      damping_factor_1: 1.0
+      damping_factor_2: 1.0
+      background_error_group: BkgError
+  - filter: Perform Action
+    filter variables:
+    - name: totalSnowDepth
+    action:
+      name: assign error
+      error parameter: 40.0
+  - filter: Variable Assignment
+    assignments:
+    - name: GrossErrorProbability/totalSnowDepth
+      type: float
+      value: 0.02
+    - name: BkgError/totalSnowDepth_background_error
+      type: float
+      value: 30.0
+

parm/land/obs/lists/gdas_land_gts_only.yaml

@@ -0,0 +1,3 @@
+observers:
+- !INC ${OBS_YAML_DIR}/adpsfc_snow.yaml
+- !INC ${OBS_YAML_DIR}/snocvr_snow.yaml

parm/land/obs/lists/gdas_land_prototype.yaml

@@ -1,3 +1,4 @@
 observers:
 - !INC ${OBS_YAML_DIR}/adpsfc_snow.yaml
+- !INC ${OBS_YAML_DIR}/snocvr_snow.yaml
 - !INC ${OBS_YAML_DIR}/ims_snow.yaml

parm/land/prep/prep_gts.yaml

@@ -2,8 +2,12 @@ gtsbufr:
   mkdir:
     - $(DATA)/obs
   copy:
-    - [$(COM_OBS)/$(OPREFIX)adpsfc.tm00.bufr_d, $(DATA)/obs/]
+    - [{{ COM_OBS }}/{{ OPREFIX }}adpsfc.tm00.bufr_d, {{ DATA }}/obs/]
+    - [{{ COM_OBS }}/{{ OPREFIX }}snocvr.tm00.bufr_d, {{ DATA }}/obs/]
 gtsioda:
   copy:
-    - [$(DATA)/$(OPREFIX)adpsfc_snow.nc4, $(COM_OBS)/$(OPREFIX)adpsfc_snow.nc4]
-
+    - [{{ DATA }}/{{ OPREFIX }}adpsfc_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}adpsfc_snow.nc4]
+    - [{{ DATA }}/{{ OPREFIX }}snocvr_snow.nc4, {{ COM_OBS }}/{{ OPREFIX }}snocvr_snow.nc4]
+bufr2ioda:
+     adpsfc: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml
+     snocvr: {{ HOMEgfs }}/sorc/gdas.cd/test/testinput/bufr_snocvr_snow.yaml

parm/soca/berror/soca_apply_steric.yaml → parm/soca/berror/old/soca_apply_steric.yaml

@@ -31,14 +31,15 @@ linear variable change:
   - linear variable change name: BkgErrFILT
     ocean_depth_min: 500  # zero where ocean is shallower than 500m
     rescale_bkgerr: 1.0   # rescale perturbation
-    efold_z: 1500.0       # Apply exponential decay
+    efold_z: 2500.0       # Apply exponential decay
   - linear variable change name: BalanceSOCA
-  trajectory:
-    state variables: [tocn, socn, uocn, vocn, ssh, hocn, layer_depth, mld]
-    date: '{{ATM_WINDOW_BEGIN}}'
-    basename: ./INPUT/
-    ocn_filename: MOM.res.nc
-    read_from_file: 1
+
+trajectory:
+  state variables: [tocn, socn, uocn, vocn, ssh, hocn, layer_depth, mld]
+  date: '{{ATM_WINDOW_BEGIN}}'
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  read_from_file: 1
 
 output increment:
   datadir: ./static_ens/

parm/soca/berror/soca_clim_ens_moments.yaml → parm/soca/berror/old/soca_clim_ens_moments.yaml

@@ -11,7 +11,6 @@ ensemble:
       basename: ./static_ens/
       ocn_filename: ocn.%mem%.nc
       ice_filename: ice.%mem%.nc
-      remap_filename: ./INPUT/MOM.res.nc
       state variables: [tocn, socn, ssh, uocn, vocn, hocn, cicen, hicen, hsnon]
     pattern: '%mem%'
     nmembers: ${CLIM_ENS_SIZE}

parm/soca/berror/soca_clim_ens_perts.yaml → parm/soca/berror/old/soca_clim_ens_perts.yaml

@@ -23,7 +23,6 @@ ensemble:
       basename: ./static_ens/
       ocn_filename: ocn.%mem%.nc
       ice_filename: ice.%mem%.nc
-      remap_filename: ./INPUT/MOM.res.nc
       state variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]
     pattern: '%mem%'
     nmembers: ${CLIM_ENS_SIZE}