From 4bf5dc24306a15307af0820b1fa41dd3591790b4 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 31 Oct 2023 11:31:01 -0500 Subject: [PATCH 1/2] Adding YAML files for Ozone in config. Update test benchmark for omi_aura. Tide up the YAMLs. --- parm/atm/obs/config/omi_aura.yaml | 73 ++++- parm/atm/obs/config/ompsnp_npp.yaml | 308 +++++++++++++++++---- parm/atm/obs/config/ompstc8_npp.yaml | 73 ++++- parm/atm/obs/testing/omi_aura.yaml | 2 +- parm/atm/obs/testing/omi_aura_noqc.yaml | 17 +- parm/atm/obs/testing/ompsnp_npp_noqc.yaml | 26 +- parm/atm/obs/testing/ompstc8_npp_noqc.yaml | 16 +- 7 files changed, 444 insertions(+), 71 deletions(-) diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index 79973780c..20cabced5 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 10000 + action: + name: reject + +# Do not use the data if row anomaly (bit 10)is 1 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityFlag + any_bit_set_of: 9 + +# Scan position check: reject scan position >= 25 +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 25 + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +# Use data with best ozone algorighm +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +obs post filters: +# GSI setup routine QC +# Gross check +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters + + diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index 5ba772fb3..c18b9abed 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -4,74 +4,284 @@ obs space: engine: type: H5File obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 + obsgrouping: + group variables: ["latitude"] + sort variable: "pressure" + sort order: "ascending" obsdataout: engine: type: H5File obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 - simulated variables: [ozoneTotal] + simulated variables: [ozoneLayer] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] -obs filters: -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 0.1 -# maxvalue: 1.1 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/total_ozone_error_flag -# minvalue: 2.1 # toss toq>2, there are some 4 and 6 -# action: -# name: reject -#- filter: BlackList -# filter variables: -# - name: ozoneTotal -# where: -# - variable: -# name: MetaData/profile_ozone_error_flag -# minvalue: 1.1 -# action: -# name: reject -- filter: BlackList - filter variables: - - name: ozoneTotal + +obs pre filters: +# Observation error assignment +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +obs prior filters: +# Do not assimilation where pressure is zero +# Zero pressure indicates the data is total column ozone +- filter: RejectList + filter variables: + - name: ozoneLayer where: - variable: name: MetaData/pressure - maxvalue: 64.00000 + maxvalue: 0.0001 + +# Sanity check on observaton values +- filter: Bounds Check + filter variables: + - name: ozoneLayer + minvalue: 0 + maxvalue: 1000 action: name: reject -- filter: Domain Check + +# Total Ozone Quality Check (keeps 0, 2) +# 0 indentifies good data +# 2 identifies good data with a solar zenith angle > 84 degrees +- filter: RejectList filter variables: - - name: ozoneTotal + - name: ozoneLayer where: - variable: - name: MetaData/latitude - minvalue: -90. - maxvalue: 90. + name: MetaData/totalOzoneQuality + is_not_in: 0, 2 + +# Profile Ozone Quality Check (keeps 0, 1, 7) +# 0 : good data +# 1 : good data with a solar zenith angle > 84 degrees +# 7 : profile for which stray light correction applied +- filter: RejectList + filter variables: + - name: ozoneLayer + where: - variable: - name: MetaData/longitude - minvalue: -180. - maxvalue: 360. -- filter: Bounds Check + name: MetaData/profileOzoneQuality + is_not_in: 0, 1, 7 + +obs post filters: +# Gross error check +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 120 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 0.001 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 30 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30000.0 + maxvalue: 110000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 40 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 20000.0 + maxvalue: 30000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 44.42 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 10100.0 + maxvalue: 20000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 57.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 6400.0 + maxvalue: 10100.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 69.4 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 4000.0 + maxvalue: 6400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 70 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 2600.0 + maxvalue: 4000.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 62.73 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1600.0 + maxvalue: 2600.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 50.52 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 1100.0 + maxvalue: 1600.0 + +- filter: Background Check filter variables: - - name: ozoneTotal - minvalue: 0.000001 - maxvalue: 1000.0 + - name: ozoneLayer + absolute threshold: 35.9 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 700.0 + maxvalue: 1100.0 + - filter: Background Check filter variables: - - name: ozoneTotal - absolute threshold: 10.0 + - name: ozoneLayer + absolute threshold: 26.41 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 400.0 + maxvalue: 700.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 20.51 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 300.0 + maxvalue: 400.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 12.82 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 200.0 + maxvalue: 300.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 10 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 70.0 + maxvalue: 200.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 5 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 40.0 + maxvalue: 70.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 2 + action: + name: reject + where: + - variable: + name: MetaData/pressure + minvalue: 30.0 + maxvalue: 40.0 + +- filter: Background Check + filter variables: + - name: ozoneLayer + absolute threshold: 1 + action: + name: reject + where: + - variable: + name: MetaData/pressure + maxvalue: 30.0 + +# End of Filters diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index e3b11ff64..545983d93 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -11,8 +11,79 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + obs operator: name: AtmVertInterpLay - geovals: [ozoneLayer] + geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] + +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +obs prior filters: +# GSI read routine QC +# range sanity check +- filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + +# Accept total_ozone_error_flag values of 0 and 1, but not any others. +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/totalOzoneQualityCode + is_not_in: 0, 1 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/bestOzoneAlgorithmFlag + is_in: 3, 13 + +# GSI setup routine QC +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + minvalue: 50.0 + +- filter: RejectList + filter variables: + - name: ozoneTotal + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 1, 2, 3, 4, 35 + - variable: + name: MetaData/latitude + maxvalue: -50.0 + +obs post filters: +- filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 10.0 + absolute threshold: 300.0 + action: + name: reject + +# End of Filters diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml index a090f5945..993095fd5 100644 --- a/parm/atm/obs/testing/omi_aura.yaml +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -85,4 +85,4 @@ obs post filters: action: name: reject -passedBenchmark: 1182 +passedBenchmark: 1170 diff --git a/parm/atm/obs/testing/omi_aura_noqc.yaml b/parm/atm/obs/testing/omi_aura_noqc.yaml index 8929c38b6..d32954a8f 100644 --- a/parm/atm/obs/testing/omi_aura_noqc.yaml +++ b/parm/atm/obs/testing/omi_aura_noqc.yaml @@ -19,9 +19,14 @@ obs operator: coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -## coef TL: 0.1 -## tolerance TL: 1.0e-13 -## tolerance AD: 1.0e-11 +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 + +passedBenchmark: 4927 # total:6082; missing:1155 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml index 219f9e942..639a8dda5 100644 --- a/parm/atm/obs/testing/ompsnp_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompsnp_npp_noqc.yaml @@ -22,11 +22,23 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [22] - -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 + +# Observation Error Assignment +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneLayer + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [0.001, 10.1325, 16.00935, 25.43258, 40.32735, 63.93607, 101.325, 160.0935, 254.3257, 403.2735, 639.3608, 1013.25, 1600.935, 2543.258, 4032.735, 6393.607, 10132.5, 16009.35, 25432.57, 40327.35, 63936.07, 101325] + errors: [7.7236, 0.02, 0.02, 0.025, 0.08, 0.15, 0.056, 0.125, 0.2, 0.299, 0.587, 0.864, 1.547, 2.718, 3.893, 4.353, 3.971, 4.407, 4.428, 3.312, 2.198, 2.285] + +passedBenchmark: 6314 # total:6314; missing:0 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 diff --git a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml index 31027007a..de9ef7839 100644 --- a/parm/atm/obs/testing/ompstc8_npp_noqc.yaml +++ b/parm/atm/obs/testing/ompstc8_npp_noqc.yaml @@ -22,11 +22,15 @@ obs operator: geovals: [mole_fraction_of_ozone_in_air] coefficients: [0.007886131] # convert from ppmv to DU nlevels: [1] +obs pre filters: +- filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 6.0 -vector ref: GsiHofXBc -tolerance: 1.e-5 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +passedBenchmark: 6693 # total:6870; missing:177 +#vector ref: GsiHofXBc +#tolerance: 1.e-5 From a217910ea4322ad204d80e33bb7c0c10636a4b41 Mon Sep 17 00:00:00 2001 From: Emily Liu Date: Tue, 31 Oct 2023 14:32:24 -0500 Subject: [PATCH 2/2] Fix comments --- parm/atm/obs/testing/omi_aura.yaml | 3 +++ parm/atm/obs/testing/ompsnp_npp.yaml | 3 +++ parm/atm/obs/testing/ompstc8_npp.yaml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/parm/atm/obs/testing/omi_aura.yaml b/parm/atm/obs/testing/omi_aura.yaml index 993095fd5..8c5bb52da 100644 --- a/parm/atm/obs/testing/omi_aura.yaml +++ b/parm/atm/obs/testing/omi_aura.yaml @@ -11,8 +11,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV omi_aura_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -85,4 +87,5 @@ obs post filters: action: name: reject +# End of Filters passedBenchmark: 1170 diff --git a/parm/atm/obs/testing/ompsnp_npp.yaml b/parm/atm/obs/testing/ompsnp_npp.yaml index f7ee9d4ec..b7974f5cd 100644 --- a/parm/atm/obs/testing/ompsnp_npp.yaml +++ b/parm/atm/obs/testing/ompsnp_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneLayer] + geovals: filename: !ENV ompsnp_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -285,4 +287,5 @@ obs post filters: name: MetaData/pressure maxvalue: 30.0 +# End of Filters passedBenchmark: 4914 diff --git a/parm/atm/obs/testing/ompstc8_npp.yaml b/parm/atm/obs/testing/ompstc8_npp.yaml index aea18a5dc..8fb3d6165 100644 --- a/parm/atm/obs/testing/ompstc8_npp.yaml +++ b/parm/atm/obs/testing/ompstc8_npp.yaml @@ -15,8 +15,10 @@ obs space: io pool: max pool size: 1 simulated variables: [ozoneTotal] + geovals: filename: !ENV ompstc8_npp_geoval_${CDATE}.nc4 + obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -91,4 +93,5 @@ obs post filters: action: name: reject +# End of Filters passedBenchmark: 6130