From 7e3423f727a9c15ac9136f44be0cc0bdd30a7db9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 31 May 2024 07:57:12 -0400 Subject: [PATCH] removed filters --- .../soca/obs/config/insitu_profile_bathy.yaml | 141 ----------- .../soca/obs/config/insitu_profile_tesac.yaml | 237 ------------------ .../config/insitu_profile_tesac_salinity.yaml | 231 ----------------- .../soca/obs/config/insitu_surface_trkob.yaml | 27 -- 4 files changed, 636 deletions(-) diff --git a/parm/soca/obs/config/insitu_profile_bathy.yaml b/parm/soca/obs/config/insitu_profile_bathy.yaml index 209e38f82..8742f44fc 100644 --- a/parm/soca/obs/config/insitu_profile_bathy.yaml +++ b/parm/soca/obs/config/insitu_profile_bathy.yaml @@ -15,144 +15,3 @@ obs operator: name: InsituTemperature obs error: covariance model: diagonal - -##-------------------------------------------------------------------------------------- -## START OF OBS FILTERS -## The QC filters used here are based on the document by IODE that can be found at -## https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf -##-------------------------------------------------------------------------------------- -#obs filters: -# -# # Land Check -# - filter: Domain Check -# where: -# - variable: {name: GeoVaLs/sea_area_fraction} -# minvalue: 0.5 -##-------------------------------------------------------------------------------------- -### Filters for T: -##-------------------------------------------------------------------------------------- -# #------------------------------------------------------------------------------------ -# ### Global range test -# #------------------------------------------------------------------------------------ -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.5 -# maxvalue: 40.0 -# -# #------------------------------------------------------------------------------------ -# ### Regional range tests -# #------------------------------------------------------------------------------------ -# -# #### Red Sea -# #------------------------------------------------------------------------------------ -# #### -# #### the document linked here describes Red sea as the are between 10N, 40E; -# #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. -# #### A more reasonable choice seemed to be a box that extends from 10 N to -# #### 30 N and 30 E to 45 East . -# #------------------------------------------------------------------------------------ -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 21.7 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 10 -# maxvalue: 30 -# - variable: -# name: MetaData/longitude -# minvalue: 30 -# -# #### Mediterranean Sea -# #----------------------------------------------------------------------------- -# ##### Area 1/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 30 -# maxvalue: 40 -# - variable: -# name: MetaData/longitude -# minvalue: -6 -# maxvalue: 40 -# ##### Area 2/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 41.5 -# - variable: -# name: MetaData/longitude -# minvalue: 20 -# maxvalue: 30 -# ##### Area 3/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 46 -# - variable: -# name: MetaData/longitude -# minvalue: 0 -# maxvalue: 20 -# -# #### Northwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.0 -# maxvalue: 24.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 50 -# maxvalue: 60 -# - variable: -# name: MetaData/longitude -# minvalue: -20 -# maxvalue: 10 -# #### Southwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.0 -# maxvalue: 30 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 25 -# maxvalue: 50 -# - variable: -# name: MetaData/longitude -# minvalue: -30 -# maxvalue: 0 -# -# #### Arctic Sea -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -1.92 -# maxvalue: 25.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 60 -# -# - filter: Background Check -# filter variables: [{name: waterTemperature}] -# threshold: 5.0 -# absolute threshold: 5.0 -# diff --git a/parm/soca/obs/config/insitu_profile_tesac.yaml b/parm/soca/obs/config/insitu_profile_tesac.yaml index 661787f68..bc8342dc8 100644 --- a/parm/soca/obs/config/insitu_profile_tesac.yaml +++ b/parm/soca/obs/config/insitu_profile_tesac.yaml @@ -18,240 +18,3 @@ obs operator: - name: InsituTemperature variables: - name: waterTemperature -# -##-------------------------------------------------------------------------------------- -## The lines below are commented out due to the composite obs operator failing looking -## for MetaData: air_pressure. -## A separate yaml file: insitu_profile_salinity.yaml is used for sailinity profiles -## -## - name: VertInterp -## observation alias file: ./obsop_name_map.yaml -## variables: -## - name: salinity -## vertical coornidate: sea_water_depth -## observation vertical coordinate: depth -## interpolation method: linear -##-------------------------------------------------------------------------------------- -##-------------------------------------------------------------------------------------- -## START OF OBS FILTERS -## The QC filters used here are based on the document by IODE that can be found at -## https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf -##-------------------------------------------------------------------------------------- -#obs filters: -# -# # Land Check -# - filter: Domain Check -# where: -# - variable: {name: GeoVaLs/sea_area_fraction} -# minvalue: 0.5 -##-------------------------------------------------------------------------------------- -### Filters for T: -##-------------------------------------------------------------------------------------- -# #------------------------------------------------------------------------------------ -# ### Global range test -# #------------------------------------------------------------------------------------ -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.5 -# maxvalue: 40.0 -# -# #------------------------------------------------------------------------------------ -# ### Regional range tests -# #------------------------------------------------------------------------------------ -# -# #### Red Sea -# #------------------------------------------------------------------------------------ -# #### -# #### the document linked here describes Red sea as the are between 10N, 40E; -# #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. -# #### A more reasonable choice seemed to be a box that extends from 10 N to -# #### 30 N and 30 E to 45 East . -# #------------------------------------------------------------------------------------ -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 21.7 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 10 -# maxvalue: 30 -# - variable: -# name: MetaData/longitude -# minvalue: 30 -# maxvalue: 45 -# -# #### Mediterranean Sea -# #----------------------------------------------------------------------------- -# ##### Area 1/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 30 -# maxvalue: 40 -# - variable: -# name: MetaData/longitude -# minvalue: -6 -# maxvalue: 40 -# ##### Area 2/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 41.5 -# - variable: -# name: MetaData/longitude -# minvalue: 20 -# maxvalue: 30 -# ##### Area 3/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: 10.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 46 -# - variable: -# name: MetaData/longitude -# minvalue: 0 -# maxvalue: 20 -# -# #### Northwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.0 -# maxvalue: 24.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 50 -# maxvalue: 60 -# - variable: -# name: MetaData/longitude -# minvalue: -20 -# maxvalue: 10 -# #### Southwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -2.0 -# maxvalue: 30 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 25 -# maxvalue: 50 -# - variable: -# name: MetaData/longitude -# minvalue: -30 -# maxvalue: 0 -# -# #### Arctic Sea -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: waterTemperature}] -# minvalue: -1.92 -# maxvalue: 25.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 60 -# -# - filter: Background Check -# filter variables: [{name: waterTemperature}] -# threshold: 5.0 -# absolute threshold: 5.0 -# -## TODO: uncomment when there is data -## #------------------------------------------------------------------------------- -## ### Spike test -## #----------------------------------------------------------------------------- -## - filter: Create Diagnostic Flags -## filter variables: -## - name: waterTemperature -## # - name: salinity -## flags: -## - name: spike -## initial value: false -## - name: step -## initial value: false -## -## - filter: Spike and Step Check -## filter variables: -## - name: ObsValue/waterTemperature -## dependent: ObsValue/waterTemperature # dy/ -## independent: MetaData/depth # dx -## count spikes: true -## count steps: true -## tolerance: -## nominal value: 10 # K nominal, in the case of temperature (not really) -## gradient: 0.1 # K/m - if dy/dx greater, could be a spike -## gradient x resolution: 10 # m - can't know dx to better precision -## factors: [1,1,0.5] # multiply tolerance, for ranges bounded by... -## x boundaries: [0,500,500] # ...these values of x (depth in m) -## boundary layer: -## x range: [0.0,300.0] # when bounded by these x values (depth in m)... -## step tolerance range: [0.0,-2.0] # ...relax tolerance for steps in boundary layer... -## maximum x interval: [50.0,100.0] # ...and ignore level if dx greater than this -## action: -## name: reject -## -## #### Count spikes -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # create derived obs value containing only T spikes -## assignments: -## - name: DerivedMetaData/waterTemperature_spikes -## type: int -## function: -## name: IntObsFunction/ProfileLevelCount -## options: -## where: -## - variable: -## name: DiagnosticFlags/spike/waterTemperature -## value: is_true -## -## #### Count steps -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # create derived obs value containing only T steps -## assignments: -## - name: DerivedMetaData/waterTemperature_steps -## type: int -## function: -## name: IntObsFunction/ProfileLevelCount -## options: -## where: -## - variable: -## name: DiagnosticFlags/step/waterTemperature -## value: is_true -## #### Count total rejections -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # compute sum 2*spikes+steps -## assignments: -## - name: DerivedMetaData/waterTemperature_rejections -## type: int -## function: -## name: IntObsFunction/LinearCombination -## options: -## variables: [DerivedMetaData/waterTemperature_spikes, DerivedMetaData/waterTemperature_steps] -## coefs: [2,1] -## #### Reject entire profile if total rejctions > threshold -## #----------------------------------------------------------------------------- -## - filter: Perform Action # reject whole profile if 2*spikes+steps>=rejection threshold -## where: -## - variable: -## name: DerivedMetaData/waterTemperature_rejections -## minvalue: 3 -## action: -## name: reject -## diff --git a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml index 9668e9c59..9190e46ef 100644 --- a/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml +++ b/parm/soca/obs/config/insitu_profile_tesac_salinity.yaml @@ -13,22 +13,6 @@ obs space: io pool: max pool size: 1 obs operator: -#---------------------------------------------------------------------------------------- -# composite obs operator is throwing an error looking for MetaData: air_pressure -# Hence the lines below and above are commented out -# name: Composite -# components: -# - name: InsituTemperature -# variables: -# - name: waterTemperature -# - name: VertInterp -# observation alias file: ./obsop_name_map.yaml -# variables: -# - name: salinity -# vertical coornidate: sea_water_depth -# observation vertical coordinate: depth -# interpolation method: linear -#--------------------------------------------------------------------------------------- name: VertInterp observation alias file: ./obsop_name_map.yaml vertical coordinate: sea_water_depth @@ -36,218 +20,3 @@ obs operator: interpolation method: linear obs error: covariance model: diagonal - -#-------------------------------------------------------------------------------------- -# START OF OBS FILTERS -# The QC filters used here are based on the document by IODE that can be found at -# https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf -#-------------------------------------------------------------------------------------- -#obs filters: -# -# # Land Check -# - filter: Domain Check -# where: -# - variable: {name: GeoVaLs/sea_area_fraction} -# minvalue: 0.5 -# -##------------------------------------------------------------------------------- -### Filters for S: -##------------------------------------------------------------------------------- -# #----------------------------------------------------------------------------- -# ### Global range test -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 41.0 -# -# #----------------------------------------------------------------------------- -# ### Regional range test -# #----------------------------------------------------------------------------- -# #### Red Sea -# #----------------------------------------------------------------------------- -# #### -# #### the document linked here describes Red sea as the are between 10N, 40E; -# #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden. -# #### A more reasonable choice seemed to be a box that extends from 10 N to -# #### 30 N and 30 E to 45 East . -# -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 41.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 10 -# maxvalue: 30 -# - variable: -# name: MetaData/longitude -# minvalue: 30 -# maxvalue: 45 -# -# #### Mediterranean Sea -# #----------------------------------------------------------------------------- -# ##### Area 1/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 30 -# maxvalue: 40 -# - variable: -# name: MetaData/longitude -# minvalue: -6 -# maxvalue: 40 -# ##### Area 2/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 41.5 -# - variable: -# name: MetaData/longitude -# minvalue: 20 -# maxvalue: 30 -# ##### Area 3/3 for Mediterranean Sea -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 40 -# maxvalue: 46 -# - variable: -# name: MetaData/longitude -# minvalue: 0 -# maxvalue: 20 -# -# -# #### Northwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 0.0 -# maxvalue: 37.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 50 -# maxvalue: 60 -# - variable: -# name: MetaData/longitude -# minvalue: -20 -# maxvalue: 10 -# -# #### Southwestern shelves -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 0.0 -# maxvalue: 38 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 25 -# maxvalue: 50 -# - variable: -# name: MetaData/longitude -# minvalue: -30 -# maxvalue: 0 -# -# #### Arctic Sea -# #----------------------------------------------------------------------------- -# - filter: Bounds Check -# filter variables: [{name: salinity}] -# minvalue: 2.0 -# maxvalue: 40.0 -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 60 -# -# - filter: Background Check -# filter variables: [{name: salinity}] -# threshold: 5.0 -# absolute threshold: 5.0 -# -## TODO: uncomment when there is data -## #------------------------------------------------------------------------------- -## ### Spike test -## #----------------------------------------------------------------------------- -## - filter: Spike and Step Check -## filter variables: -## - name: ObsValue/salinity -## dependent: ObsValue/salinity # dy/ -## independent: MetaData/depth # dx -## count spikes: true -## count steps: true -## tolerance: -## nominal value: 1.0 # PSU nominal, in the case of salinity (not really) -## threshold: 0.6 # weird salinity thing -## factors: [1,1,0.2] # multiply tolerance, for ranges bounded by... -## x boundaries: [0,200,600] # ...these values of x (depth in m) -## boundary layer: -## x range: [0.0,300.0] # when bounded by these x values (depth in m)... -## maximum x interval: [50.0,100.0] # ...and ignore level if dx greater than this -## action: -## name: reject -## -## #### Count spikes -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # create derived obs value containing only S spikes -## assignments: -## - name: DerivedMetaData/salinity_spikes -## type: int -## function: -## name: IntObsFunction/ProfileLevelCount -## options: -## where: -## - variable: -## name: DiagnosticFlags/spike/salinity -## value: is_true -## #### Count steps -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # create derived obs value containing only S steps -## assignments: -## - name: DerivedMetaData/salinity_steps -## type: int -## function: -## name: IntObsFunction/ProfileLevelCount -## options: -## where: -## - variable: -## name: DiagnosticFlags/step/salinity -## value: is_true -## #### Count total rejections -## #----------------------------------------------------------------------------- -## - filter: Variable Assignment # compute sum 2*spikes+steps -## assignments: -## - name: DerivedMetaData/salinity_rejections -## type: int -## function: -## name: IntObsFunction/LinearCombination -## options: -## variables: [DerivedMetaData/salinity_spikes, DerivedMetaData/salinity_steps] -## coefs: [2,1] -## #### Reject entire profile if total rejctions > threshold -## #----------------------------------------------------------------------------- -## - filter: Perform Action # reject whole profile if 2*spikes+steps>=rejection threshold -## where: -## - variable: -## name: DerivedMetaData/salinity_rejections -## minvalue: 3 -## action: -## name: reject -## #------------------------------------------------------------------------------- -## ### End of Spike test -## #----------------------------------------------------------------------------- diff --git a/parm/soca/obs/config/insitu_surface_trkob.yaml b/parm/soca/obs/config/insitu_surface_trkob.yaml index d60a03bf3..e35230223 100644 --- a/parm/soca/obs/config/insitu_surface_trkob.yaml +++ b/parm/soca/obs/config/insitu_surface_trkob.yaml @@ -16,30 +16,3 @@ obs operator: observation alias file: ./obsop_name_map.yaml obs error: covariance model: diagonal - -#obs filters: -#- filter: Domain Check -# where: -# - variable: {name: GeoVaLs/sea_area_fraction} -# minvalue: 0.9 -#- filter: Bounds Check -# minvalue: 1.0 -# maxvalue: 41.0 -#- filter: Background Check -# threshold: 5.0 -#- filter: Domain Check -# where: -# - variable: {name: ObsError/seaSurfaceTemperature} -# minvalue: 1.0e-8 -#- filter: Domain Check -# where: -# - variable: { name: GeoVaLs/sea_ice_area_fraction} -# maxvalue: 1.0e-5 -#- filter: Domain Check -# where: -# - variable: {name: GeoVaLs/sea_surface_temperature} -# minvalue: -1.0 -#- filter: Domain Check -# where: -# - variable: {name: GeoVaLs/distance_from_coast} -# minvalue: 100e3