gdas_validation merge

NOAA-EMC · Jan 23, 2024 · 5e34fc0 · 5e34fc0
2 parents 086cff3 + 99c431a
commit 5e34fc0
Show file tree

Hide file tree

Showing 30 changed files with 1,174 additions and 210 deletions.
diff --git a/parm/aero/hofx/hofx_nomodel.yaml b/parm/aero/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-window length: $(AERO_WINDOW_LENGTH)
+time window:
+  begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+  length: $(AERO_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml

diff --git a/parm/aero/variational/3dvar_fgat_gfs_aero.yaml b/parm/aero/variational/3dvar_fgat_gfs_aero.yaml
@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-FGAT
-  window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-  window length: $(AERO_WINDOW_LENGTH)
+  time window:
+    begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+    length: $(AERO_WINDOW_LENGTH)
   analysis variables: &3dvars
                       [mass_fraction_of_sulfate_in_air,
                        mass_fraction_of_hydrophobic_black_carbon_in_air,

diff --git a/parm/aero/variational/3dvar_gfs_aero.yaml b/parm/aero/variational/3dvar_gfs_aero.yaml
@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-Var
-  window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-  window length: $(AERO_WINDOW_LENGTH)
+  time window:
+    begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+    length: $(AERO_WINDOW_LENGTH)
   analysis variables: &3dvars
                       [mass_fraction_of_sulfate_in_air,
                        mass_fraction_of_hydrophobic_black_carbon_in_air,

diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 forecast length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:

diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml

diff --git a/parm/atm/hofx/hofx_ufotest.yaml b/parm/atm/hofx/hofx_ufotest.yaml
@@ -1,3 +1,4 @@
-window begin: '{{ATM_WINDOW_BEGIN}}'
-window end: '{{ATM_WINDOW_END}}'
+time window:
+  begin: '{{ATM_WINDOW_BEGIN}}'
+  end: '{{ATM_WINDOW_END}}'
 observations: !INC ${OBS_LIST}
diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml
@@ -10,8 +10,9 @@ geometry:
   npy: $(npy_ges)
   npz: $(npz_ges)
   field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
 
 background:

diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml
@@ -3,11 +3,11 @@ obs space:
   obsdatain:
     engine: 
       type: H5File
-      obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/obs/$(OPREFIX)conventional_ps.prepbufr.nc
   obsdataout:
     engine:
       type: H5File
-      obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4
+      obsfile: $(DATA)/diags/diag_conventional_ps_prepbufr_{{ current_cycle | to_YMDH }}.nc
   io pool:
     max pool size: 1
   simulated variables: [stationPressure]
@@ -21,6 +21,11 @@ obs operator:
   geovar_sfc_geomz: surface_altitude
   geovar_geomz: geopotential_height
 
+linear obs operator:
+  name: Identity
+  variables:
+  - name: stationPressure
+
 obs prior filters:
 # Initial Error Assignments for SFC Observations 
 - filter: Perform Action
@@ -103,36 +108,107 @@ obs prior filters:
         xvals:  [80000, 75000, 70000, 65000, 60000, 55000  ]
         errors: [  110,   120,   120,   120,   120, 1.0e+11]
 
-obs post filters:
-# Observation range sanity check
-- filter: Bounds Check
-  filter variables:
-  - name: stationPressure
-  minvalue: 37499.0
-  maxvalue: 106999.0
-  action:
-    name: reject
+# Create PreQC group variable (pqm in GSI read_prepbufr)
+- filter: Variable Assignment
+  assignments:
+  - name: InputObsError/stationPressure
+    type: float 
+    source variable: ObsErrorData/stationPressure
 
-# Reject all ObsType 183
-- filter: RejectList
+# Set observation quality-realted variables 
+# Create PreQC group variable (pqm in GSI read_prepbufr)
+- filter: Variable Assignment
+  assignments:
+  - name: PreQC/stationPressure
+    type: int 
+    source variable: QualityMarker/stationPressure
+
+# Create PreUseFlag group variable (usage in GSI read_prepbufr)
+# Initialize
+- filter: Variable Assignment
+  assignments:
+  - name: PreUseFlag/stationPressure
+    type: int 
+    source variable: PreQC/stationPressure  
+
+- filter: Variable Assignment
+  where:
+  - variable:
+      name: PreUseFlag/stationPressure
+    is_in: 1-15 
+  assignments:
+  - name: PreUseFlag/stationPressure
+    value: 0  
+# Re-assignment
+- filter: Variable Assignment
   where:
   - variable:
       name: ObsType/stationPressure
     is_in: 183
+  assignments:
+  - name: PreUseFlag/stationPressure
+    value: 100 
 
-# Reject surface pressure below 500 hPa 
-- filter: Bounds Check
-  filter variables:
-  - name: stationPressure
-  minvalue: 50000.00
-  action:
-    name: reject
+- filter: Variable Assignment
+  where:
+  - variable:
+      name:  ObsValue/stationPressure
+    is_defined: 
+  - variable:
+      name:  ObsValue/stationPressure
+    maxvalue: 50000.00
+  where operator: and
+  assignments:
+  - name: PreUseFlag/stationPressure
+    value: 100 
 
-- filter: RejectList
+- filter: Variable Assignment
   where:
   - variable:
       name: PreQC/stationPressure
-    is_in: 4-15
+    is_in: 9, 12, 15 
+  assignments:
+  - name: PreUseFlag/stationPressure
+    value: 100 
+
+- filter: Variable Assignment
+  where:
+  - variable:
+      name: PreQC/stationPressure
+    is_in: 4-15 
+  assignments:
+  - name: PreUseFlag/stationPressure
+    value: 101 
+
+## Observation range sanity check
+#- filter: Bounds Check
+#  filter variables:
+#  - name: stationPressure
+#  minvalue: 37499.0
+#  maxvalue: 106999.0
+#  action:
+#    name: reject
+
+## Reject all ObsType 183
+#- filter: RejectList
+#  where:
+#  - variable:
+#      name: ObsType/stationPressure
+#    is_in: 183
+
+## Reject surface pressure below 500 hPa 
+#- filter: Bounds Check
+#  filter variables:
+#  - name: stationPressure
+#  minvalue: 50000.00
+#  action:
+#    name: reject
+
+#- filter: RejectList
+#  where:
+#  - variable:
+#      name: PreQC/stationPressure
+#    is_in: 4-15
 
 # Inflate obs error based on obs type
 - filter: Perform Action 
@@ -145,6 +221,7 @@ obs post filters:
     name: inflate error
     inflation factor: 1.2  
 
+obs post filters:
 # Calculate obs error inflation factors for duplicated observations at the same location
 - filter: Variable Assignment
   assignments:
@@ -172,26 +249,6 @@ obs post filters:
     name: inflate error
     inflation factor: 0.7
 
-# Reduce original observation error based on obs type and subtype
-# In this case: reduce original obs error for buoy 
-- filter: Variable Assignment
-  where:
-  - variable:
-      name: ObsType/stationPressure
-    is_in: 180
-  - variable:
-      name: ObsSubType/stationPressure
-    is_in: 0
-  assignments:
-  - name: ObsError/stationPressure
-    type: float
-    function:
-      name: ObsFunction/Arithmetic 
-      options: 
-        variables:
-        - name: ObsError/stationPressure
-        coefs: [0.7]
-
 # Calculate obs error inflation factors for large discrepancies between model and observations 
 - filter: Variable Assignment
   assignments:
@@ -290,6 +347,5 @@ obs post filters:
     is_not_in: 0, 1
   action:
     name: reject 
-
 # End of Filters 
 
diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml
@@ -22,4 +22,5 @@ observers:
 ##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml
 ##- !INC ${OBS_YAML_DIR}/sfc.yaml
 ##- !INC ${OBS_YAML_DIR}/sfcship.yaml
+##- !INC ${OBS_YAML_DIR}/conv_ps.yaml
 ##- !INC ${OBS_YAML_DIR}/gnssro.yaml
diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml
@@ -104,15 +104,14 @@ obs prior filters:
         xvals:  [80000, 75000, 70000, 65000, 60000, 55000  ]
         errors: [  110,   120,   120,   120,   120, 1.0e+11]
 
-obs post filters:
-# Observation range sanity check
-- filter: Bounds Check
-  filter variables:
-  - name: stationPressure
-  minvalue: 37499.0
-  maxvalue: 106999.0
-  action:
-    name: reject
+## Observation range sanity check
+#- filter: Bounds Check
+#  filter variables:
+#  - name: stationPressure
+#  minvalue: 37499.0
+#  maxvalue: 106999.0
+#  action:
+#    name: reject
 
 # Reject all ObsType 183
 - filter: RejectList
@@ -121,19 +120,19 @@ obs post filters:
       name: ObsType/stationPressure
     is_in: 183
 
-# Reject surface pressure below 500 hPa 
-- filter: Bounds Check
-  filter variables:
-  - name: stationPressure
-  minvalue: 50000.00
-  action:
-    name: reject
+## Reject surface pressure below 500 hPa 
+#- filter: Bounds Check
+#  filter variables:
+#  - name: stationPressure
+#  minvalue: 50000.00
+#  action:
+#    name: reject
 
-- filter: RejectList
-  where:
-  - variable:
-      name: PreQC/stationPressure
-    is_in: 4-15
+#- filter: RejectList
+#  where:
+#  - variable:
+#      name: PreQC/stationPressure
+#    is_in: 4-15
 
 # Inflate obs error based on obs type
 - filter: Perform Action 
@@ -146,6 +145,7 @@ obs post filters:
     name: inflate error
     inflation factor: 1.2  
 
+obs post filters:
 # Calculate obs error inflation factors for duplicated observations at the same location
 - filter: Variable Assignment
   assignments:
@@ -173,26 +173,6 @@ obs post filters:
     name: inflate error
     inflation factor: 0.7
 
-# Reduce original observation error based on obs type and subtype
-# In this case: reduce original obs error for buoy 
-- filter: Variable Assignment
-  where:
-  - variable:
-      name: ObsType/stationPressure
-    is_in: 180
-  - variable:
-      name: ObsSubType/stationPressure
-    is_in: 0
-  assignments:
-  - name: ObsError/stationPressure
-    type: float
-    function:
-      name: ObsFunction/Arithmetic 
-      options: 
-        variables:
-        - name: ObsError/stationPressure
-        coefs: [0.7]
-
 # Calculate obs error inflation factors for large discrepancies between model and observations 
 - filter: Variable Assignment
   assignments:

diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml
@@ -3,6 +3,7 @@ cost function:
   time window:
     begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
     length: $(ATM_WINDOW_LENGTH)
+    bound to include: begin 
   analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
   geometry:
     fms initialization:

diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json
@@ -1,11 +1,11 @@
 {
-    "data_format": "prepbufr",
-    "data_type": "prepbufr",
-    "cycle_type": "{{ RUN }}",
-    "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
-    "dump_directory": "{{ DMPDIR }}",
-    "ioda_directory": "{{ COM_OBS }}",
-    "subsets": [ "ADPUPA" ],
-    "data_provider": "U.S. NOAA",
-    "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS",
+  "data_format"     : "prepbufr",
+  "data_type"       : "ADPUPA",
+  "cycle_type"      : "{{ RUN }}",
+  "cycle_datetime"  : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"  : "{{ DMPDIR }}",
+  "ioda_directory"  : "{{ COM_OBS }}",
+  "subsets"         : [ "ADPUPA" ],
+  "data_provider"   : "U.S. NOAA",
+  "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS"
 }
diff --git a/parm/land/hofx/hofx_nomodel.yaml b/parm/land/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
-window length: $(LAND_WINDOW_LENGTH)
+time window:
+  begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
+  length: $(LAND_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml