From ea9b6d201f38b3bc48ddd14440b4503d75c8ae69 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:47:15 -0500 Subject: [PATCH 01/22] Switch to std::vector in Rads2Ioda.h (#747) ### Description #### This PR includes what is done below - Switch to ```std::vector``` in RadsToIoda.h for replacing the array - Implement type of ```short sla``` to ```std::vector sla``` close https://github.com/NOAA-EMC/GDASApp/issues/664 #### ioda results are below ``` netcdf rads_adt_3b_2021181.ioda { dimensions: Location = 11 ; variables: int Location(Location) ; Location:suggested_chunk_dim = 11LL ; // global attributes: string :_ioda_layout = "ObsGroup" ; :_ioda_layout_version = 0 ; string :obs_source_files = "rads_adt_3b_2021181.nc4" ; string :mission_index = "CRYOSAT2 = 6 JASON-1 = 3 JASON-2 = 4 JASON-3 = 5 SARAL = 7 SNTNL-3A = 1 SNTNL-3B = 2 " ; string :references = "RADS Data Manual, Version 4.2 or later" ; data: Location = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; group: MetaData { variables: int cycle(Location) ; cycle:_FillValue = -2147483643 ; int64 dateTime(Location) ; dateTime:_FillValue = -9223372036854775801LL ; string dateTime:units = "seconds since 1858-11-17T00:00:00Z" ; float latitude(Location) ; latitude:_FillValue = -3.368795e+38f ; float longitude(Location) ; longitude:_FillValue = -3.368795e+38f ; float mdt(Location) ; mdt:_FillValue = -3.368795e+38f ; int mission(Location) ; mission:_FillValue = -2147483643 ; int oceanBasin(Location) ; oceanBasin:_FillValue = -2147483643 ; int pass(Location) ; pass:_FillValue = -2147483643 ; data: cycle = 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54 ; dateTime = 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384, 5131728384 ; latitude = 59.73273, 59.5059, 58.76763, 58.71078, 58.65391, 58.59704, 58.54016, 58.48327, 58.42637, 58.36946, 58.31255 ; longitude = 163.4174, 163.2622, 162.7704, 162.7333, 162.6964, 162.6595, 162.6228, 162.5861, 162.5496, 162.5132, 162.4768 ; mdt = 0.1927, 0.2235, 0.4589, 0.5115, 0.5307, 0.5154999, 0.4983, 0.4831, 0.4875, 0.4722, 0.4527 ; mission = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ; oceanBasin = -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ; pass = 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232 ; } // group MetaData group: ObsError { variables: float absoluteDynamicTopography(Location) ; absoluteDynamicTopography:_FillValue = -3.368795e+38f ; data: absoluteDynamicTopography = 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 ; } // group ObsError group: ObsValue { variables: float absoluteDynamicTopography(Location) ; absoluteDynamicTopography:_FillValue = -3.368795e+38f ; data: absoluteDynamicTopography = 0.6505, 0.7307, 0.6026, 0.5871, 0.5561, 0.5246, 0.4981, 0.4661, 0.4391, 0.4409, 0.4283 ; } // group ObsValue group: PreQC { variables: int absoluteDynamicTopography(Location) ; absoluteDynamicTopography:_FillValue = -2147483643 ; data: absoluteDynamicTopography = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; } // group PreQC } ``` --- utils/obsproc/Rads2Ioda.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h index f7ea4f69e..7234d6c12 100644 --- a/utils/obsproc/Rads2Ioda.h +++ b/utils/obsproc/Rads2Ioda.h @@ -44,17 +44,17 @@ namespace gdasapp { gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, floatMetadataNames, intMetadataNames); // Read non-optional metadata: datetime, longitude and latitude - int lat[iodaVars.location_]; // NOLINT - ncFile.getVar("lat").getVar(lat); + std::vector lat(iodaVars.location_); + ncFile.getVar("lat").getVar(lat.data()); - int lon[iodaVars.location_]; // NOLINT - ncFile.getVar("lon").getVar(lon); + std::vector lon(iodaVars.location_); + ncFile.getVar("lon").getVar(lon.data()); float geoscaleFactor; ncFile.getVar("lon").getAtt("scale_factor").getValues(&geoscaleFactor); - float datetime[iodaVars.location_]; // NOLINT - ncFile.getVar("time_mjd").getVar(datetime); + std::vector datetime(iodaVars.location_); + ncFile.getVar("time_mjd").getVar(datetime.data()); iodaVars.referenceDate_ = "seconds since 1858-11-17T00:00:00Z"; std::map altimeterMap; @@ -84,10 +84,10 @@ namespace gdasapp { iodaVars.strGlobalAttr_["references"] = references; // Read optional integer metadata "pass" and "cycle" - int pass[iodaVars.location_]; // NOLINT - ncFile.getVar("pass").getVar(pass); - int cycle[iodaVars.location_]; // NOLINT - ncFile.getVar("cycle").getVar(cycle); + std::vector pass(iodaVars.location_); + ncFile.getVar("pass").getVar(pass.data()); + std::vector cycle(iodaVars.location_); + ncFile.getVar("cycle").getVar(cycle.data()); // Store optional metadata, set ocean basins to -999 for now for (int i = 0; i < iodaVars.location_; i++) { @@ -95,14 +95,14 @@ namespace gdasapp { } // Get adt_egm2008 obs values and attributes - int adt[iodaVars.location_]; // NOLINT - ncFile.getVar("adt_egm2008").getVar(adt); + std::vector adt(iodaVars.location_); + ncFile.getVar("adt_egm2008").getVar(adt.data()); float scaleFactor; ncFile.getVar("adt_egm2008").getAtt("scale_factor").getValues(&scaleFactor); // Read sla - short sla[iodaVars.location_]; // NOLINT - ncFile.getVar("sla").getVar(sla); + std::vector sla(iodaVars.location_); + ncFile.getVar("sla").getVar(sla.data()); // Update non-optional Eigen arrays for (int i = 0; i < iodaVars.location_; i++) { From 302eda5c5771035f09dbf268b2df903110dd8b38 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:44:20 -0500 Subject: [PATCH 02/22] Correct sensor amsua_n19 typo (#751) The `Useflag check` in `amsua_n19.yaml` specified the wrong sensor. The string `amaua_n19` was used instead of `amsua_n19`. The typo is corrected in [`feature/amsua_n19`](https://github.com/NOAA-EMC/GDASApp/tree/feature/amsua_n19). Fixes #726 --- parm/atm/obs/config/amsua_n19.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index df6c38528..be40d9650 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -357,7 +357,7 @@ obs filters: - name: ObsFunction/ChannelUseflagCheckRad channels: *amsua_n19_channels options: - sensor: amaua_n19 + sensor: amsua_n19 channels: *amsua_n19_channels use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, From 4f089c4d72483dd2f88bca69df11143db8f2ee72 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:54:39 -0500 Subject: [PATCH 03/22] Fix a bug for dateTime in IcecAmsr2Ioda.h (#756) ### Description - Implemented an additional trim function to ensure that the Obs.Values align with their respective dTime entries - Minor changes within for loop and removed unnecessary function - Only applied to amsr2_icec --------- Co-authored-by: Guillaume Vernieres --- utils/obsproc/IcecAmsr2Ioda.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h index 414d25017..0295b6e37 100644 --- a/utils/obsproc/IcecAmsr2Ioda.h +++ b/utils/obsproc/IcecAmsr2Ioda.h @@ -64,10 +64,9 @@ namespace gdasapp { ncFile.getVar("Scan_Time").getVar(oneTmpdateTimeVal.data()); iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; + size_t index = 0; std::tm timeinfo = {}; for (int i = 0; i < ntimes; i += dimTimeSize) { - for (int j = 0; j < dimTimeSize && i + j < ntimes; j++) { - } timeinfo.tm_year = oneTmpdateTimeVal[i] - 1900; // Year since 1900 timeinfo.tm_mon = oneTmpdateTimeVal[i + 1] - 1; // 0-based; 8 represents Sep timeinfo.tm_mday = oneTmpdateTimeVal[i + 2]; @@ -77,20 +76,22 @@ namespace gdasapp { // Calculate and store the seconds since the Unix epoch time_t epochtime = std::mktime(&timeinfo); - iodaVars.datetime_(i/6) = static_cast(epochtime); + iodaVars.datetime_(index) = static_cast(epochtime); + index++; } // Update non-optional Eigen arrays for (int i = 0; i < iodaVars.location_; i++) { iodaVars.longitude_(i) = oneDimLonVal[i]; iodaVars.latitude_(i) = oneDimLatVal[i]; - iodaVars.obsVal_(i) = static_cast(oneDimObsVal[i]*0.01); + iodaVars.obsVal_(i) = static_cast(oneDimObsVal[i]*0.01f); iodaVars.obsError_(i) = 0.1; // Do something for obs error iodaVars.preQc_(i) = oneDimFlagsVal[i]; } // basic test for iodaVars.trim - Eigen::Array mask = (iodaVars.obsVal_ >= 0.0); + Eigen::Array mask = (iodaVars.obsVal_ >= 0.0 + && iodaVars.datetime_ > 0.0); iodaVars.trim(mask); return iodaVars; From 94abd9aaa6b1960018eabc427212208fcb7a46bc Mon Sep 17 00:00:00 2001 From: BrettHoover-NOAA <98188219+BrettHoover-NOAA@users.noreply.github.com> Date: Mon, 27 Nov 2023 07:50:27 -0600 Subject: [PATCH 04/22] Added SEVIRI/METEOSAT-11 satwinds to end-to-end validation (#767) Small code-change to include SEVIRI/METEOSAT-11 satwinds in end-to-end validation. This uses the hook-ins already provided by the SEVIRI/METEOSAT-8 satwinds code that was merged in https://github.com/NOAA-EMC/GDASApp/pull/764. New file: parm/atm/obs/config/satwind_seviri_m11.yaml - this is nearly a direct copy of parm/atm/obs/config/satwind_seviri_m8.yaml, since they utilize the same filters. Only the header information has changed. Modified file: parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json - METEOSAT-11 was added to the `satellite_info` group End-to-end testing looks very similar to what was seen for SEVIRI/METEOSAT-8 satwinds, detailed in https://github.com/NOAA-EMC/GDASApp/issues/758. A brief run-down: (LW)IR winds are in QC agreement down to a 0.02% difference (cloud-top)WV winds are in QC agreement down to a 0.20% difference VIS winds differ by 4.5%, but there are only 22 accepted winds in GSI and 23 in JEDI, the difference is a single satwind All ob, HofX, and ob-minus-HofX differences look good Increment summary, from gdasatmanlrun.log: ``` 0: ---------------------------------------------------------------------------------------------------- 0: Increment print | number of fields = 8 | cube sphere face size: C768 0: eastward_wind | Min:-3.479765e-01 Max:+3.430433e-01 RMS:+4.990993e-04 0: northward_wind | Min:-4.226316e-01 Max:+3.376561e-01 RMS:+4.889429e-04 0: air_temperature | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: surface_pressure | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: specific_humidity | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: cloud_liquid_ice | Min:+0.000000e+00 Max:+1.618770e-20 RMS:+1.293217e-23 0: cloud_liquid_water | Min:+0.000000e+00 Max:+1.474788e-19 RMS:+2.167418e-22 0: ozone_mass_mixing_ratio | Min:+0.000000e+00 Max:+0.000000e+00 RMS:+0.000000e+00 0: ---------------------------------------------------------------------------------------------------- ``` Co-authored-by: Brett Hoover --- parm/atm/obs/config/satwind_seviri_m11.yaml | 405 ++++++++++++++++++ .../bufr2ioda_satwind_amv_seviri.json | 1 + 2 files changed, 406 insertions(+) create mode 100644 parm/atm/obs/config/satwind_seviri_m11.yaml diff --git a/parm/atm/obs/config/satwind_seviri_m11.yaml b/parm/atm/obs/config/satwind_seviri_m11.yaml new file mode 100644 index 000000000..763e59fda --- /dev/null +++ b/parm/atm/obs/config/satwind_seviri_m11.yaml @@ -0,0 +1,405 @@ +obs space: + name: satwind_seviri_m11 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.seviri_m11.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_seviri_m11_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of SEVIRI/METEOSAT-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #758 for details: https://github.com/NOAA-EMC/GDASApp/issues/758 +#obs pre filters: +#- filter: Gaussian Thinning +# where: +# - variable: ObsType/windEastward +# is_in: 243, 253 +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 254 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 254 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + + +# Exclude data over non-water surface type where latitude > 20N for Type 253 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# EUMET VIS (243) reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# EUMET WV (254) reject when pressure greater than 399 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json index fd90d4a02..3fad3c258 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json @@ -11,5 +11,6 @@ "sensor_info" : { "sensor_name": "SEVIRI", "sensor_full_name": "Spinning Enhanced Visible and InfraRed Imager", "sensor_id": 999 }, "satellite_info" : [ { "satellite_name": "M8", "satellite_full_name": "METEOSAT-8", "satellite_id": 55, "launch time": "YYYYMMDD" }, + { "satellite_name": "M11", "satellite_full_name": "METEOSAT-11", "satellite_id": 70, "launch time": "YYYYMMDD" }, ] } From e7fc7cb3e4c9a574fa549fc3fed28beda0544f01 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:31:56 -0500 Subject: [PATCH 05/22] A few updates for ATMS end-to-end testing (#768) In [ previous PR](https://github.com/NOAA-EMC/GDASApp/pull/757) for ATMS, the end-to-end testing was done without data thinning. And the excessive QC filtering in clear-sky areas, compared to GSI, was found. (See [plots](https://github.com/NOAA-EMC/GDASApp/pull/757#issuecomment-1819753244)) This PR includes two fixes for ATMS in the end-to-end testing and one new feature: 1. fix the excess QC filtering in clear-sky areas. ---> This has a paring [UFO PR #3122](https://github.com/JCSDA-internal/ufo/pull/3122) 2. add data thinning 3. add the diagnostic flags (QC) --- **This can [reproduce QC flags](https://github.com/NOAA-EMC/GDASApp/pull/768#issuecomment-1828209921) from GSI in UFO** **To test updates in this PR, please check out the UFO branch: [feature/satrad](https://github.com/JCSDA-internal/ufo/tree/feature/satrad) from JCSDA-internal in gdas-validation. This branch consolidates all proposed code changes to UFO for gdas validation: UFO PR #https://github.com/JCSDA-internal/ufo/pull/3122 UFO PR #https://github.com/JCSDA-internal/ufo/pull/3121 UFO PR #https://github.com/JCSDA-internal/ufo/pull/3094 --------- Co-authored-by: Cory Martin --- parm/atm/obs/config/atms_n20.yaml | 63 ++++++++++---- parm/atm/obs/config/atms_npp.yaml | 63 ++++++++++---- parm/atm/obs/testing/atms_n20.yaml | 111 ++++++++++++++---------- parm/atm/obs/testing/atms_n20_noqc.yaml | 21 +++-- parm/atm/obs/testing/atms_npp.yaml | 110 +++++++++++++---------- parm/atm/obs/testing/atms_npp_noqc.yaml | 21 +++-- 6 files changed, 242 insertions(+), 147 deletions(-) diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 93590ab7a..2a29f5d95 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian @@ -56,7 +57,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature @@ -83,6 +83,9 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: # Step 0-B: Calculate derived variables @@ -224,7 +227,7 @@ obs post filters: channels: *atms_n20_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -235,7 +238,7 @@ obs post filters: name: InitialObsError@DerivedMetaData channels: *atms_n20_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -250,15 +253,16 @@ obs post filters: - name: reject # Step 2: Data Thinning -#- filter: Gaussian Thinning -# horizontal_mesh: 145 -# use_reduced_horizontal_grid: true -## round_horizontal_bin_count_to_nearest: true -## partition_longitude_bins_using_mesh: true -# actions: -# - name: set -# flag: Thinning -# - name: reject +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -286,7 +290,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -338,7 +342,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData @@ -361,7 +365,7 @@ obs post filters: name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_n20_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData @@ -383,7 +387,7 @@ obs post filters: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_n20_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData @@ -409,7 +413,7 @@ obs post filters: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *atms_n20_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData @@ -504,7 +508,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -518,7 +522,7 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 @@ -528,3 +532,24 @@ obs post filters: ignore: rejected observations - name: reject +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml index bb668ab9d..ecddd346c 100644 --- a/parm/atm/obs/config/atms_npp.yaml +++ b/parm/atm/obs/config/atms_npp.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID atms_npp EndianType: little_endian @@ -56,7 +57,6 @@ obs bias: obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature @@ -83,6 +83,9 @@ obs pre filters: - name: InterChannelConsistency initial value: false force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false obs post filters: # Step 0-B: Calculate derived variables @@ -224,7 +227,7 @@ obs post filters: channels: *atms_npp_channels coefs: [1, -1] -# Step 0-C: Assign initial all-sky observation error +# Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature @@ -235,7 +238,7 @@ obs post filters: name: InitialObsError@DerivedMetaData channels: *atms_npp_channels -# Step 1: Remove observations from the edge of the scan +# Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: - name: brightnessTemperature @@ -250,15 +253,16 @@ obs post filters: - name: reject # Step 2: Data Thinning -#- filter: Gaussian Thinning -# horizontal_mesh: 145 -# use_reduced_horizontal_grid: true -## round_horizontal_bin_count_to_nearest: true -## partition_longitude_bins_using_mesh: true -# actions: -# - name: set -# flag: Thinning -# - name: reject +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -286,7 +290,7 @@ obs post filters: flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -338,7 +342,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData @@ -361,7 +365,7 @@ obs post filters: name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_npp_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData @@ -383,7 +387,7 @@ obs post filters: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_npp_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData @@ -409,7 +413,7 @@ obs post filters: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *atms_npp_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData @@ -504,7 +508,7 @@ obs post filters: ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -518,7 +522,7 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 @@ -528,3 +532,24 @@ obs post filters: ignore: rejected observations - name: reject +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/testing/atms_n20.yaml b/parm/atm/obs/testing/atms_n20.yaml index 9f1c913b0..006a452f3 100644 --- a/parm/atm/obs/testing/atms_n20.yaml +++ b/parm/atm/obs/testing/atms_n20.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_n20 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_n20 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_n20_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *atms_n20_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency initial value: false - force reinitialization: true + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,6 +262,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Variable Assignment @@ -302,7 +299,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -458,7 +455,6 @@ obs post filters: 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature @@ -468,7 +464,7 @@ obs post filters: channels: *atms_n20_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -486,14 +482,37 @@ obs post filters: sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck ignore: rejected observations - name: reject + passedBenchmark: 181401 # GSI: 181403 (difference is in channel 7) diff --git a/parm/atm/obs/testing/atms_n20_noqc.yaml b/parm/atm/obs/testing/atms_n20_noqc.yaml index 1d9a9dae1..95c2463a9 100644 --- a/parm/atm/obs/testing/atms_n20_noqc.yaml +++ b/parm/atm/obs/testing/atms_n20_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_n20 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_n20 obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_n20_satbias_${GDATE}.nc4 variational bc: diff --git a/parm/atm/obs/testing/atms_npp.yaml b/parm/atm/obs/testing/atms_npp.yaml index beae69095..c83fc7b8f 100644 --- a/parm/atm/obs/testing/atms_npp.yaml +++ b/parm/atm/obs/testing/atms_npp.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_npp - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_npp obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_npp_channels 1-22 + geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_npp_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *atms_npp_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck initial value: false - force reinitialization: true + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,6 +262,10 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Variable Assignment @@ -302,7 +299,7 @@ obs post filters: maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -467,7 +464,7 @@ obs post filters: channels: *atms_npp_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -484,15 +481,38 @@ obs post filters: use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + passedBenchmark: 182751 # GSI: 182751 diff --git a/parm/atm/obs/testing/atms_npp_noqc.yaml b/parm/atm/obs/testing/atms_npp_noqc.yaml index f8afbb175..5bbf45307 100644 --- a/parm/atm/obs/testing/atms_npp_noqc.yaml +++ b/parm/atm/obs/testing/atms_npp_noqc.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_npp - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_npp obsdatain: @@ -19,8 +10,20 @@ obs space: obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] channels: &atms_npp_channels 1-22 + geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_npp_satbias_${GDATE}.nc4 variational bc: From efac95c6f81838494a24e9563cdf6cc23595fa88 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 28 Nov 2023 11:13:26 -0500 Subject: [PATCH 06/22] Add an option to point to the "enkfgdas" backgrounds instead of the pre-generated 30 member ensemble (#771) #### Work done Mainly what the title says. The default ctest is now assuming that `DOHYBVAR`=`YES`, I don't really think it's worth it to cover the `DOHYBVAR`=`NO` in the ctest. I may change my mind, but in both cases, `soca` is configured to use the hybrid envar solver, the only difference is the origin of the ensemble members. #### Work left to be done We need to define an ensemble "COM_OCEAN/ICE_HISTORY_PREV" environment variable in the prep jjob (in the g-w). Flagged as a `TODO` in the code. --- parm/soca/berror/saber_blocks.yaml | 2 +- parm/soca/berror/soca_ensb.yaml | 2 +- ...exgdas_global_marine_analysis_bmat_vrfy.sh | 80 ------------------- scripts/exgdas_global_marine_analysis_prep.py | 63 +++++++++++---- test/soca/gw/CMakeLists.txt | 1 - test/soca/gw/prep.sh | 18 +++++ test/soca/gw/run_jjobs.yaml.test | 2 + ush/soca/run_jjobs.py | 8 ++ 8 files changed, 77 insertions(+), 99 deletions(-) delete mode 100755 scripts/exgdas_global_marine_analysis_bmat_vrfy.sh diff --git a/parm/soca/berror/saber_blocks.yaml b/parm/soca/berror/saber_blocks.yaml index 9487ddae3..b8965cf05 100644 --- a/parm/soca/berror/saber_blocks.yaml +++ b/parm/soca/berror/saber_blocks.yaml @@ -60,7 +60,7 @@ components: ice_filename: 'ice.%mem%.nc' state variables: [tocn, socn, ssh, uocn, vocn, cicen, hicen, hsnon] pattern: '%mem%' - nmembers: ${CLIM_ENS_SIZE} + nmembers: ${ENS_SIZE} localization: localization method: SABER saber central block: diff --git a/parm/soca/berror/soca_ensb.yaml b/parm/soca/berror/soca_ensb.yaml index 447d5b5fd..42e054531 100644 --- a/parm/soca/berror/soca_ensb.yaml +++ b/parm/soca/berror/soca_ensb.yaml @@ -17,7 +17,7 @@ vertical geometry: read_from_file: 3 soca increments: - number of increments: ${CLIM_ENS_SIZE} + number of increments: ${ENS_SIZE} pattern: '%mem%' template: date: '{{ATM_WINDOW_BEGIN}}' diff --git a/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh b/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh deleted file mode 100755 index 4903d7d40..000000000 --- a/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgdas_global_marine_analysis_bmat_vrfy.sh -# Script description: Diagnose the JEDI/SOCA B-matrix -# -# Author: Guillaume Vernieres Org: NCEP/EMC Date: 2023-01-12 -# -# Abstract: This script runs the JEDI/SOCA dirac application with the B-matrix -# used in the variational step -# $Id$ -# -# Attributes: -# Language: POSIX shell -# Machine: Orion -# -################################################################################ - -# Set environment. -export VERBOSE=${VERBOSE:-"YES"} -if [ $VERBOSE = "YES" ]; then - echo $(date) EXECUTING $0 $* >&2 - set -x -fi - -# Directories -pwd=$(pwd) - -# Utilities -export NLN=${NLN:-"/bin/ln -sf"} -export DIRAC="${HOMEgfs}/sorc/gdas.cd/ush/ufsda/dirac_yaml.py" - -################################################################################ -# Compute the impulse response of the B-matrix from the variational application -# field = 1: tocn -# field = 2: socn -# field = 3: ssh -# field = 4: cicen -# field = 5: hicen - -arr=("tocn" "socn" "ssh" "cicen" "hicen") -level=1 -ndiracs=100 - -for i in "${!arr[@]}" -do - var=${arr[i]} - ifield=$((i+1)) - # generate dirac yamls -cat > dirac_output.yaml << EOL -datadir: ./Data -exp: dirac_${var}_${level} -type: an -EOL - - ${DIRAC} --varyaml 'var.yaml' \ - --fields 'soca_gridspec.nc' \ - --dim1 'xaxis_1' \ - --dim2 'yaxis_1' \ - --diracyaml 'dirac.yaml' \ - --ndiracs ${ndiracs} \ - --level ${level} \ - --fieldindex ${ifield} \ - --diracoutput dirac_output.yaml - export err=$? - - # run the dirac application - $APRUN_OCNANAL $JEDI_BIN/soca_dirac.x dirac.yaml - export err=$? - if [ $err -gt 0 ]; then - exit $err - fi -done - -################################################################################ - -exit ${err} - -################################################################################ diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index 4e0635be2..aaeadc612 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -89,6 +89,10 @@ def cice_hist2fms(input_filename, output_filename): # open the CICE history file ds = xr.open_dataset(input_filename) + if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables: + logging.info(f"*** Already reformatted, skipping.") + return + # rename the dimensions to xaxis_1 and yaxis_1 ds = ds.rename({'ni': 'xaxis_1', 'nj': 'yaxis_1'}) @@ -224,6 +228,11 @@ def find_clim_ens(input_date): comin_obs = os.getenv('COMIN_OBS') # R2D2 DB for now anl_dir = os.getenv('DATA') staticsoca_dir = os.getenv('SOCA_INPUT_FIX_DIR') +if os.getenv('DOHYBVAR') == "YES": + dohybvar = True + nmem_ens = int(os.getenv('NMEM_ENS')) +else: + dohybvar = False # create analysis directories diags = os.path.join(anl_dir, 'diags') # output dir for soca DA obs space @@ -242,6 +251,7 @@ def find_clim_ens(input_date): fcst_begin = datetime.strptime(os.getenv('PDY')+os.getenv('cyc'), '%Y%m%d%H') RUN = os.getenv('RUN') cyc = os.getenv('cyc') +gcyc = os.getenv('gcyc') PDY = os.getenv('PDY') ################################################################################ @@ -310,21 +320,43 @@ def find_clim_ens(input_date): FileHandler({'copy': bkgerr_list}).sync() ################################################################################ -# stage static ensemble - -logging.info(f"---------------- Stage climatological ensemble") -clim_ens_member_list = [] -clim_ens_dir = find_clim_ens(pytz.utc.localize(window_begin, is_dst=None)) -clim_ens_size = len(glob.glob(os.path.abspath(os.path.join(clim_ens_dir, 'ocn.*.nc')))) -os.environ['CLIM_ENS_SIZE'] = str(clim_ens_size) - -for domain in ['ocn', 'ice']: - for mem in range(1, clim_ens_size+1): - fname = domain+"."+str(mem)+".nc" - fname_in = os.path.abspath(os.path.join(clim_ens_dir, fname)) - fname_out = os.path.abspath(os.path.join(static_ens, fname)) - clim_ens_member_list.append([fname_in, fname_out]) -FileHandler({'copy': clim_ens_member_list}).sync() +# stage ensemble members +if dohybvar: + logging.info("---------------- Stage ensemble members") + nmem_ens = int(os.getenv('NMEM_ENS')) + longname = {'ocn': 'ocean', 'ice': 'ice'} + ens_member_list = [] + for mem in range(1, nmem_ens+1): + for domain in ['ocn', 'ice']: + # TODO(Guillaume): make use and define ensemble COM in the j-job + ensdir = os.path.join(os.getenv('COM_OCEAN_HISTORY_PREV'), '..', '..', '..', '..', '..', + f'enkf{RUN}.{PDY}', f'{gcyc}', f'mem{str(mem).zfill(3)}', + 'model_data', longname[domain], 'history') + ensdir = os.path.normpath(ensdir) + f009 = f'enkfgdas.t{gcyc}z.{domain}f009.nc' + + fname_in = os.path.abspath(os.path.join(ensdir, f009)) + fname_out = os.path.abspath(os.path.join(static_ens, domain+"."+str(mem)+".nc")) + ens_member_list.append([fname_in, fname_out]) + FileHandler({'copy': ens_member_list}).sync() + + # reformat the cice history output + for mem in range(1, nmem_ens+1): + cice_fname = os.path.abspath(os.path.join(static_ens, "ice."+str(mem)+".nc")) + cice_hist2fms(cice_fname, cice_fname) +else: + logging.info("---------------- Stage offline ensemble members") + ens_member_list = [] + clim_ens_dir = find_clim_ens(pytz.utc.localize(window_begin, is_dst=None)) + nmem_ens = len(glob.glob(os.path.abspath(os.path.join(clim_ens_dir, 'ocn.*.nc')))) + for domain in ['ocn', 'ice']: + for mem in range(1, nmem_ens+1): + fname = domain+"."+str(mem)+".nc" + fname_in = os.path.abspath(os.path.join(clim_ens_dir, fname)) + fname_out = os.path.abspath(os.path.join(static_ens, fname)) + ens_member_list.append([fname_in, fname_out]) + FileHandler({'copy': ens_member_list}).sync() +os.environ['ENS_SIZE'] = str(nmem_ens) ################################################################################ # prepare JEDI yamls @@ -456,7 +488,6 @@ def find_clim_ens(input_date): else: logging.info(f"using default SABER blocks yaml") os.environ['SABER_BLOCKS_YAML'] = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'saber_blocks.yaml') -os.environ['CLIM_ENS_SIZE'] = str(clim_ens_size) # substitute templated variables in the var config logging.info(f"{config}") diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 575bc7f2b..c5a3a26ca 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -39,7 +39,6 @@ file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch) # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_* set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT" -# "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY" "JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT" "JGDAS_GLOBAL_OCEAN_ANALYSIS_POST" diff --git a/test/soca/gw/prep.sh b/test/soca/gw/prep.sh index a3590752b..2aeadf224 100755 --- a/test/soca/gw/prep.sh +++ b/test/soca/gw/prep.sh @@ -60,3 +60,21 @@ ice_files=$(ls ${clim_ens_dir}/ice.*.nc) for ice_file in ${ice_files}; do ncrename -O -d ni,xaxis_1 -d nj,yaxis_1 -v aice_h,aicen -v hi_h,hicen -v hs_h,hsnon ${ice_file} done + +# Invent ensemble forecast for f009 +fake_ocean_members=`ls` +fake_ice_members=`ls ${clim_ens_dir/ice.*.nc}` +COMENS=${project_binary_dir}/test/soca/gw/COM/enkfgdas.20180415 +for mem in {1..4} +do + echo "member: $mem" + # ocean member + oceandir=${COMENS}/06/mem00${mem}/model_data/ocean/history + mkdir -p $oceandir + cp ${clim_ens_dir}/ocn.${mem}.nc $oceandir/enkfgdas.t06z.ocnf009.nc + echo ${clim_ens_dir}/ocn.${mem}.nc $oceandir/enkfgdas.t06z.ocnf009.nc + # ice member + icedir=${COMENS}/06/mem00${mem}/model_data/ice/history + mkdir -p $icedir + cp ${clim_ens_dir}/ice.${mem}.nc $icedir/enkfgdas.t06z.icef009.nc +done diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index f908c841f..3270d820a 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -43,6 +43,8 @@ setup_expt config: DO_JEDIOCNVAR: "YES" DO_JEDILANDDA: "NO" DO_MERGENSST: "NO" + NMEM_ENS: "4" + DOHYBVAR: "YES" ocnanal: SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static CASE_ANL: C48 diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index d4149633c..80d8a3c54 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -190,6 +190,14 @@ def copy_bkgs(self): self.f.write(f"cp {com_ice_history_src}/*icef*.nc $COM_ICE_HISTORY_PREV \n") self.f.write(f"cp {com_ice_restart_src}/*cice_model*.nc $COM_ICE_RESTART_PREV \n") + # copy ensemble members + ensbkgs = os.path.join(self.com_src, f"enkf{self.RUN}.{self.gPDY}") + if os.path.exists(os.path.join(ensbkgs, self.gcyc)): + self.f.write(f"cp -r {ensbkgs} $ROTDIR \n") + else: + print('Aborting, ensemble backgrounds not found') + sys.exit() + def fixconfigs(self): """ Replace cone of the env. var. in the configs From bc55a0118cac53462b7dbe0e65af24242404e15f Mon Sep 17 00:00:00 2001 From: Shastri Paturi Date: Tue, 28 Nov 2023 20:38:32 +0000 Subject: [PATCH 07/22] BUFR2IODA python API converter for SUBPFL ARGO profiles (#762) ## Description This PR adds a bufr2ioda python API converter for marine SUBPFL ARGO profiles based on the work done during the "End-to-End Code Sprint" by the atmosphere group. Closes #736 --------- Co-authored-by: Cory Martin --- .../bufr2ioda_subpfl_argo_profiles.json | 12 + .../bufr2ioda_subpfl_argo_profiles.py | 323 ++++++++++++++++++ ush/ioda/bufr2ioda/run_bufr2ioda.py | 2 +- 3 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.json create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.py diff --git a/parm/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.json b/parm/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.json new file mode 100644 index 000000000..2d2460288 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.json @@ -0,0 +1,12 @@ +{ + "data_format" : "subpfl", + "subsets" : "SUBPFL", + "source" : "NCEP data tank", + "data_type" : "argo", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "data_description" : "6-hourly in situ ARGO profiles", + "data_provider" : "U.S. NOAA" +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.py b/ush/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.py new file mode 100644 index 000000000..329a4fe9e --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_subpfl_argo_profiles.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import numpy as np +import numpy.ma as ma +import os +import argparse +import math +import calendar +import time +import copy +from datetime import datetime +import json +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + + +def Compute_sequenceNumber(lon): + lon_u, seqNum = np.unique(lon, return_inverse=True) + seqNum = seqNum.astype(np.int64) + logger.debug(f"Len of Sequence Number: {len(seqNum)}") + + return seqNum + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + data_format = config["data_format"] + source = config["source"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + cycle_datetime = config["cycle_datetime"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + # General Information + converter = 'BUFR to IODA Converter' + platform_description = 'ARGO profiles from ' + subsets + ': temperature and salinity' + bufrfile = f"{cycle_type}.t{hh}z.{data_format}.tm00.bufr_d" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), f"atmos", + bufrfile) + + # =========================================== + # Make the QuerySet for all the data we want + # =========================================== + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet() + + # MetaData + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('ryear', '*/RCYR') + q.add('rmonth', '*/RCMO') + q.add('rday', '*/RCDY') + q.add('rhour', '*/RCHR') + q.add('rminute', '*/RCMI') + q.add('stationID', '*/WMOP') + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('depth', '*/GLPFDATA/WPRES') + + # ObsValue + q.add('temp', '*/GLPFDATA/SSTH') + q.add('saln', '*/GLPFDATA/SALNH') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # =============================================================== + # Open the BUFR file and execute the QuerySet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + logger.debug(f" ... Executing QuerySet: get MetaData ...") + dateTime = r.get_datetime('year', 'month', 'day', 'hour', 'minute', group_by='depth') + dateTime = dateTime.astype(np.int64) + rcptdateTime = r.get_datetime('ryear', 'rmonth', 'rday', 'rhour', 'rminute', group_by='depth') + rcptdateTime = rcptdateTime.astype(np.int64) + stationID = r.get('stationID', group_by='depth') + lat = r.get('latitude', group_by='depth') + lon = r.get('longitude', group_by='depth') + depth = r.get('depth', group_by='depth') + # convert depth in pressure units to meters (rho * g * h) + depth = np.float32(depth.astype(float) * 0.0001) + + # ObsValue + logger.debug(f" ... Executing QuerySet: get ObsValue ...") + temp = r.get('temp', group_by='depth') + temp -= 273.15 + saln = r.get('saln', group_by='depth') + + # print ("masking temp & saln ...") + mask = ((temp > -10.0) & (temp <= 50.0)) & ((saln >= 0.0) & (saln <= 45.0)) + temp = temp[mask] + lat = lat[mask] + lon = lon[mask] + depth = depth[mask] + stationID = stationID[mask] + + logger.debug(f"Get sequenceNumber based on unique longitude...") + seqNum = Compute_sequenceNumber(lon) + + # ================================================= + # Separate ARGO profiles from subpfl tank + # ================================================= + logger.debug(f"Finding index for ARGO floats where the second number of the stationID = 9...") + index_list = [] + for index, number in enumerate(stationID): + # Convert the number to a string + number_str = str(number) + + # Check if the second character is equal to '9' + if number_str[1] == '9': + index_list.append(index) + logger.debug(f"Indexing Done...") + + # Apply index + stationID = stationID[index_list] + lat = lat[index_list] + lon = lon[index_list] + depth = depth[index_list] + temp = temp[index_list] + saln = saln[index_list] + seqNum = seqNum[index_list] + dateTime = dateTime[index_list] + rcptdateTime = rcptdateTime[index_list] + + # ObsError + logger.debug(f"Generating ObsError array with constant value (instrument error)...") + ObsError_temp = np.float32(np.ma.masked_array(np.full((len(index_list)), 0.02))) + ObsError_saln = np.float32(np.ma.masked_array(np.full((len(index_list)), 0.01))) + + # PreQC + logger.debug(f"Generating PreQC array with 0...") + PreQC = (np.ma.masked_array(np.full((len(index_list)), 0))).astype(np.int32) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") +# #================================ +# # Check values of BUFR variables, dimension and type +# #================================ + + logger.debug(f" temp min, max, length, dtype = {temp.min()}, {temp.max()}, {len(temp)}, {temp.dtype}") + logger.debug(f" saln min, max, length, dtype = {saln.min()}, {saln.max()}, {len(saln)}, {saln.dtype}") + logger.debug(f" lon min, max, length, dtype = {lon.min()}, {lon.max()}, {len(lon)}, {lon.dtype}") + logger.debug(f" lat min, max, length, dtype = {lat.min()}, {lat.max()}, {len(lat)}, {lat.dtype}") + logger.debug(f" depth min, max, length, dtype = {depth.min()}, {depth.max()}, {len(depth)}, {depth.dtype}") + logger.debug(f" PreQC min, max, length, dtype = {PreQC.min()}, {PreQC.max()}, {len(PreQC)}, {PreQC.dtype}") + logger.debug(f" ObsError_temp min, max, length, dtype = {ObsError_temp.min()}, {ObsError_temp.max()}, {len(ObsError_temp)}, {ObsError_temp.dtype}") + logger.debug(f" ObsError_saln min, max, length, dtype = {ObsError_saln.min()}, {ObsError_saln.max()}, {len(ObsError_saln)}, {ObsError_saln.dtype}") + + logger.debug(f" stationID shape, dtype = {stationID.shape}, {stationID.astype(str).dtype}") + logger.debug(f" dateTime shape, dtype = {dateTime.shape}, {dateTime.dtype}") + logger.debug(f" rcptdateTime shape, dytpe = {rcptdateTime.shape}, {rcptdateTime.dtype}") + logger.debug(f" sequence Num shape, dtype = {seqNum.shape}, {seqNum.dtype}") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Create the dimensions + dims = {'Location': np.arange(0, lat.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}_profiles.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('Converter', converter) + obsspace.write_attr('source', source) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeRange', [str(dateTime.min()), str(dateTime.max())]) + obsspace.write_attr('platformLongDescription', platform_description) + +# Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, and attributes") + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(dateTime) + + # rcptDatetime + obsspace.create_var('MetaData/rcptdateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'receipt Datetime') \ + .write_data(rcptdateTime) + + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon.dtype, fillval=lon.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat) + + # Station Identification + obsspace.create_var('MetaData/stationID', dtype=stationID.dtype, fillval=stationID.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stationID) + + # Depth + obsspace.create_var('MetaData/depth', dtype=depth.dtype, fillval=depth.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Water depth') \ + .write_data(depth) + + # Sequence Number + obsspace.create_var('MetaData/sequenceNumber', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'Sequence Number') \ + .write_data(seqNum) + + # PreQC + obsspace.create_var('PreQC/waterTemperature', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + obsspace.create_var('PreQC/salinity', dtype=PreQC.dtype, fillval=PreQC.fill_value) \ + .write_attr('long_name', 'PreQC') \ + .write_data(PreQC) + + # ObsError + obsspace.create_var('ObsError/waterTemperature', dtype=ObsError_temp.dtype, fillval=ObsError_temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_temp) + + obsspace.create_var('ObsError/salinity', dtype=ObsError_saln.dtype, fillval=ObsError_saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('long_name', 'ObsError') \ + .write_data(ObsError_saln) + + # ObsValue + obsspace.create_var('ObsValue/waterTemperature', dtype=temp.dtype, fillval=temp.fill_value) \ + .write_attr('units', 'degC') \ + .write_attr('valid_range', np.array([-10.0, 50.0], dtype=np.float32)) \ + .write_attr('long_name', 'Temperature below surface') \ + .write_data(temp) + + obsspace.create_var('ObsValue/salinity', dtype=saln.dtype, fillval=saln.fill_value) \ + .write_attr('units', 'psu') \ + .write_attr('valid_range', np.array([0.0, 45.0], dtype=np.float32)) \ + .write_attr('long_name', 'Salinity below surface') \ + .write_data(saln) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA: {running_time} \ + seconds") + + logger.debug(f"All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_subpfl_argo_profiles.py', level=log_level, + colored_log=True) + + with open(config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 34df54b5f..4d826359e 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -31,7 +31,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Specify observation types to be processed by a script BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", - "gpsro_bufr", "conventional_prepbufr_ps"] + "gpsro_bufr", "conventional_prepbufr_ps", "bufr2ioda_subpfl_argo_profiles.py"] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") From 88125cad26bac198da5d11feb6c6cf505a4ccdd1 Mon Sep 17 00:00:00 2001 From: Mindo Choi <141867620+apchoiCMD@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:31:15 -0500 Subject: [PATCH 08/22] Add the feature of comparing variables to reference files (#772) #### Work added - add the `testoutput` method into base class - now be able to see a basic stat within `ctest` --- utils/obsproc/Ghrsst2Ioda.h | 5 +-- utils/obsproc/IcecAmsr2Ioda.h | 1 + utils/obsproc/NetCDFToIodaConverter.h | 6 ++- utils/obsproc/Rads2Ioda.h | 1 + utils/obsproc/Smap2Ioda.h | 1 + utils/obsproc/Smos2Ioda.h | 1 + utils/test/CMakeLists.txt | 4 ++ utils/test/testinput/gdas_ghrsst2ioda.yaml | 1 - utils/test/testinput/gdas_icecamsr2ioda.yaml | 8 +++- utils/test/testinput/gdas_rads2ioda.yaml | 8 +++- utils/test/testinput/gdas_smap2ioda.yaml | 5 +++ utils/test/testinput/gdas_smos2ioda.yaml | 5 +++ utils/test/testref/ghrsst2ioda.test | 46 +++++--------------- utils/test/testref/icecamsr2ioda.test | 26 +++++++++++ utils/test/testref/rads2ioda.test | 26 +++++++++++ utils/test/testref/smap2ioda.test | 26 +++++++++++ utils/test/testref/smos2ioda.test | 26 +++++++++++ 17 files changed, 152 insertions(+), 44 deletions(-) create mode 100644 utils/test/testref/icecamsr2ioda.test create mode 100644 utils/test/testref/rads2ioda.test create mode 100644 utils/test/testref/smap2ioda.test create mode 100644 utils/test/testref/smos2ioda.test diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h index aaf2a0645..d0a611822 100644 --- a/utils/obsproc/Ghrsst2Ioda.h +++ b/utils/obsproc/Ghrsst2Ioda.h @@ -43,7 +43,7 @@ namespace gdasapp { // Open the NetCDF file in read-only mode netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); - oops::Log::test() << "Reading " << fileName << std::endl; + oops::Log::info() << "Reading... " << fileName << std::endl; // Get number of obs int dimLon = ncFile.getDim("lon").getSize(); int dimLat = ncFile.getDim("lat").getSize(); @@ -211,9 +211,6 @@ namespace gdasapp { int64_t mean = iodaVars.datetime_.sum() / iodaVars.datetime_.size(); iodaVars.datetime_.setConstant(mean); - // Test output - iodaVars.testOutput(); - return iodaVars; }; }; // class Ghrsst2Ioda diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h index 0295b6e37..61bacfade 100644 --- a/utils/obsproc/IcecAmsr2Ioda.h +++ b/utils/obsproc/IcecAmsr2Ioda.h @@ -31,6 +31,7 @@ namespace gdasapp { // Open the NetCDF file in read-only mode netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); + oops::Log::info() << "Reading... " << fileName << std::endl; // Get the number of obs in the file int dimxSize = ncFile.getDim("Number_of_X_Dimension").getSize(); diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h index d7d20be7c..ecc6b6aa6 100644 --- a/utils/obsproc/NetCDFToIodaConverter.h +++ b/utils/obsproc/NetCDFToIodaConverter.h @@ -63,6 +63,7 @@ namespace gdasapp { iodaVars.append(providerToIodaVars(inputFilenames_[i])); oops::Log::info() << " appending: " << inputFilenames_[i] << std::endl; oops::Log::info() << " obs count: " << iodaVars.location_ << std::endl; + oops::Log::test() << "Reading: " << inputFilenames_ << std::endl; } nobs = iodaVars.location_; @@ -101,7 +102,7 @@ namespace gdasapp { ioda::Variable iodaDatetime = ogrp.vars.createWithScales("MetaData/dateTime", {ogrp.vars["Location"]}, long_params); - iodaDatetime.atts.add("units", {iodaVars.referenceDate_}, {1}); + iodaDatetime.atts.add("units", {iodaVarsAll.referenceDate_}, {1}); ioda::Variable iodaLat = ogrp.vars.createWithScales("MetaData/latitude", {ogrp.vars["Location"]}, float_params); @@ -148,6 +149,9 @@ namespace gdasapp { count++; } + // Test output + iodaVars.testOutput(); + // Write obs info to group oops::Log::info() << "Writing ioda file" << std::endl; iodaLon.writeWithEigenRegular(iodaVarsAll.longitude_); diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h index 7234d6c12..cff4bea07 100644 --- a/utils/obsproc/Rads2Ioda.h +++ b/utils/obsproc/Rads2Ioda.h @@ -30,6 +30,7 @@ namespace gdasapp { // Open the NetCDF file in read-only mode netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); + oops::Log::info() << "Reading... " << fileName << std::endl; // Get the number of obs in the file int nobs = ncFile.getDim("time").getSize(); diff --git a/utils/obsproc/Smap2Ioda.h b/utils/obsproc/Smap2Ioda.h index 0a2bc5a66..be3c2d8e3 100644 --- a/utils/obsproc/Smap2Ioda.h +++ b/utils/obsproc/Smap2Ioda.h @@ -30,6 +30,7 @@ namespace gdasapp { // Open the NetCDF file in read-only mode netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); + oops::Log::info() << "Reading... " << fileName << std::endl; // Get number of obs int dim0 = ncFile.getDim("phony_dim_0").getSize(); diff --git a/utils/obsproc/Smos2Ioda.h b/utils/obsproc/Smos2Ioda.h index caf04a76d..9368240b3 100644 --- a/utils/obsproc/Smos2Ioda.h +++ b/utils/obsproc/Smos2Ioda.h @@ -29,6 +29,7 @@ namespace gdasapp { // Open the NetCDF file in read-only mode netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); + oops::Log::info() << "Reading... " << fileName << std::endl; // Get number of obs int nobs = ncFile.getDim("n_grid_points").getSize(); diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index 73a73385e..96aff4631 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -10,6 +10,10 @@ list( APPEND utils_test_input set( gdas_utils_test_ref testref/ghrsst2ioda.test + testref/rads2ioda.test + testref/smap2ioda.test + testref/smos2ioda.test + testref/icecamsr2ioda.test ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput) diff --git a/utils/test/testinput/gdas_ghrsst2ioda.yaml b/utils/test/testinput/gdas_ghrsst2ioda.yaml index 703f75a81..35eea2a30 100644 --- a/utils/test/testinput/gdas_ghrsst2ioda.yaml +++ b/utils/test/testinput/gdas_ghrsst2ioda.yaml @@ -1,7 +1,6 @@ provider: GHRSST window begin: 2018-04-15T06:00:00Z window end: 2018-04-15T12:00:00Z -variable: seaSurfaceTemperature binning: stride: 2 min number of obs: 1 diff --git a/utils/test/testinput/gdas_icecamsr2ioda.yaml b/utils/test/testinput/gdas_icecamsr2ioda.yaml index 0d2b0dc4f..28809c99b 100644 --- a/utils/test/testinput/gdas_icecamsr2ioda.yaml +++ b/utils/test/testinput/gdas_icecamsr2ioda.yaml @@ -1,6 +1,12 @@ provider: AMSR2 window begin: 2018-04-15T06:00:00Z window end: 2018-04-15T12:00:00Z -output file: icec_amsr2_north_1.ioda.nc +output file: icec_amsr2_north.ioda.nc input files: - icec_amsr2_north_1.nc4 +- icec_amsr2_north_2.nc4 + +test: + reference filename: testref/icecamsr2ioda.test + test output filename: testoutput/icecamsr2ioda.test + float relative tolerance: 1e-6 diff --git a/utils/test/testinput/gdas_rads2ioda.yaml b/utils/test/testinput/gdas_rads2ioda.yaml index 556c09f5b..16f0be711 100644 --- a/utils/test/testinput/gdas_rads2ioda.yaml +++ b/utils/test/testinput/gdas_rads2ioda.yaml @@ -1,7 +1,13 @@ provider: RADS window begin: 2018-04-15T06:00:00Z window end: 2018-04-15T12:00:00Z -output file: rads_adt_3b_2021181.ioda.nc +output file: rads_adt_3ab_2021181.ioda.nc #ocean basin: RECCAP2_region_masks_all_v20221025.nc input files: +- rads_adt_3a_2021181.nc4 - rads_adt_3b_2021181.nc4 + +test: + reference filename: testref/rads2ioda.test + test output filename: testoutput/rads2ioda.test + float relative tolerance: 1e-6 diff --git a/utils/test/testinput/gdas_smap2ioda.yaml b/utils/test/testinput/gdas_smap2ioda.yaml index ec617c5f0..944e4faac 100644 --- a/utils/test/testinput/gdas_smap2ioda.yaml +++ b/utils/test/testinput/gdas_smap2ioda.yaml @@ -5,3 +5,8 @@ output file: sss_smap.ioda.nc input files: - sss_smap_1.nc4 - sss_smap_2.nc4 + +test: + reference filename: testref/smap2ioda.test + test output filename: testoutput/smap2ioda.test + float relative tolerance: 1e-6 diff --git a/utils/test/testinput/gdas_smos2ioda.yaml b/utils/test/testinput/gdas_smos2ioda.yaml index e7a960c85..cdae09e59 100644 --- a/utils/test/testinput/gdas_smos2ioda.yaml +++ b/utils/test/testinput/gdas_smos2ioda.yaml @@ -5,3 +5,8 @@ output file: sss_smos.ioda.nc input files: - sss_smos_1.nc4 - sss_smos_2.nc4 + +test: + reference filename: testref/smos2ioda.test + test output filename: testoutput/smos2ioda.test + float relative tolerance: 1e-6 diff --git a/utils/test/testref/ghrsst2ioda.test b/utils/test/testref/ghrsst2ioda.test index c8e895e1e..35e2ed25f 100644 --- a/utils/test/testref/ghrsst2ioda.test +++ b/utils/test/testref/ghrsst2ioda.test @@ -1,52 +1,26 @@ -Reading ghrsst_sst_mb_202107010000.nc4 +Reading: [ghrsst_sst_mb_202107010000.nc4,ghrsst_sst_mb_202107010100.nc4] seconds since 1981-01-01T00:00:00Z obsVal: Min: 276.708 - Max: 276.9 - Sum: 4982.63 -obsError: - Min: 0.32 - Max: 0.32 - Sum: 5.76 -preQc: - Min: 0 - Max: 0 - Sum: 0 -longitude: - Min: 131.52 - Max: 131.71 - Sum: 2369.13 -latitude: - Min: -55.5 - Max: -55.42 - Sum: -998.28 -datetime: - Min: 1277942528 - Max: 1277942528 - Sum: 23002965504 -Reading ghrsst_sst_mb_202107010100.nc4 -seconds since 1981-01-01T00:00:00Z -obsVal: - Min: 281.547 Max: 281.714 - Sum: 5069.06 + Sum: 10051.7 obsError: - Min: 0.58 + Min: 0.32 Max: 0.61 - Sum: 10.8225 + Sum: 16.5825 preQc: Min: 0 Max: 0 Sum: 0 longitude: - Min: 151.12 + Min: 131.52 Max: 151.31 - Sum: 2721.93 + Sum: 5091.06 latitude: - Min: 56.62 + Min: -55.5 Max: 56.7 - Sum: 1019.88 + Sum: 21.6 datetime: - Min: 1277946624 + Min: 1277942528 Max: 1277946624 - Sum: 23003039232 + Sum: 46006004736 diff --git a/utils/test/testref/icecamsr2ioda.test b/utils/test/testref/icecamsr2ioda.test new file mode 100644 index 000000000..a8ef1c959 --- /dev/null +++ b/utils/test/testref/icecamsr2ioda.test @@ -0,0 +1,26 @@ +Reading: [icec_amsr2_north_1.nc4,icec_amsr2_north_2.nc4] +seconds since 1970-01-01T00:00:00Z +obsVal: + Min: 0 + Max: 1 + Sum: 11.91 +obsError: + Min: 0.1 + Max: 0.1 + Sum: 1.7 +preQc: + Min: 0 + Max: 8 + Sum: 40 +longitude: + Min: -168.867 + Max: 148.766 + Sum: -302.819 +latitude: + Min: 72.8259 + Max: 86.8975 + Sum: 1346.66 +datetime: + Min: 1625066557 + Max: 1625126209 + Sum: 27626797699 diff --git a/utils/test/testref/rads2ioda.test b/utils/test/testref/rads2ioda.test new file mode 100644 index 000000000..f6e0283ce --- /dev/null +++ b/utils/test/testref/rads2ioda.test @@ -0,0 +1,26 @@ +iReading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4] +seconds since 1858-11-17T00:00:00Z +obsVal: + Min: 0.1671 + Max: 0.7307 + Sum: 8.3145 +obsError: + Min: 0.1 + Max: 0.1 + Sum: 2.2 +preQc: + Min: 0 + Max: 0 + Sum: 0 +longitude: + Min: -21.761 + Max: 163.417 + Sum: 1552.07 +latitude: + Min: -43.9731 + Max: 59.7327 + Sum: 165.588 +datetime: + Min: 5131727872 + Max: 5131728384 + Sum: 112898018816 diff --git a/utils/test/testref/smap2ioda.test b/utils/test/testref/smap2ioda.test new file mode 100644 index 000000000..c644a5324 --- /dev/null +++ b/utils/test/testref/smap2ioda.test @@ -0,0 +1,26 @@ +Reading: [sss_smap_1.nc4,sss_smap_2.nc4] +seconds since 1970-01-01T00:00:00Z +obsVal: + Min: 31.8368 + Max: 37.4915 + Sum: 1042.15 +obsError: + Min: 0.481312 + Max: 2.62376 + Sum: 29.5826 +preQc: + Min: 0 + Max: 643 + Sum: 787 +longitude: + Min: -94.3724 + Max: -41.5933 + Sum: -2180.65 +latitude: + Min: -56.7806 + Max: 40.8618 + Sum: -370.853 +datetime: + Min: 1625197568 + Max: 1625204736 + Sum: 48756048640 diff --git a/utils/test/testref/smos2ioda.test b/utils/test/testref/smos2ioda.test new file mode 100644 index 000000000..8ec093709 --- /dev/null +++ b/utils/test/testref/smos2ioda.test @@ -0,0 +1,26 @@ +Reading: [sss_smos_1.nc4,sss_smos_2.nc4] +seconds since 1970-01-01T00:00:00Z +obsVal: + Min: 0.994212 + Max: 38.3212 + Sum: 1587.47 +obsError: + Min: 0.508775 + Max: 213.003 + Sum: 294.419 +preQc: + Min: 56 + Max: 999 + Sum: 20696 +longitude: + Min: -86.052 + Max: 137.108 + Sum: 840.527 +latitude: + Min: -64.333 + Max: 77.959 + Sum: 411.168 +datetime: + Min: 1625087616 + Max: 1625093440 + Sum: 78004317504 From ee520cc7f05b8ac6f96374f26afc42d0bece04f9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 30 Nov 2023 09:37:39 -0500 Subject: [PATCH 09/22] Fix to the time interpolation yaml key in the soca obs yaml configuration (#778) - fixes #777 --- parm/soca/obs/config/adt_3a_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_3b_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_6a_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_all.yaml | 4 ++-- parm/soca/obs/config/adt_c2_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_coperl4.yaml | 2 -- parm/soca/obs/config/adt_j2.yaml | 2 ++ parm/soca/obs/config/adt_j2_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_j3.yaml | 4 ++-- parm/soca/obs/config/adt_j3_egm2008.yaml | 4 ++-- parm/soca/obs/config/adt_sa_egm2008.yaml | 4 ++-- parm/soca/obs/config/icec_amsr2_north.yaml | 4 ++-- parm/soca/obs/config/icec_amsr2_south.yaml | 4 ++-- parm/soca/obs/config/icec_emc.yaml | 4 ++-- parm/soca/obs/config/icec_nsidc_nh.yaml | 4 ++-- parm/soca/obs/config/icec_nsidc_sh.yaml | 4 ++-- parm/soca/obs/config/icec_ssmis_f17_north.yaml | 4 ++-- parm/soca/obs/config/icec_ssmis_f17_south.yaml | 4 ++-- parm/soca/obs/config/icec_ssmis_f18_north.yaml | 4 ++-- parm/soca/obs/config/icec_ssmis_f18_south.yaml | 4 ++-- parm/soca/obs/config/icefb_gdr.yaml | 4 ++-- parm/soca/obs/config/sss_smap.yaml | 4 ++-- parm/soca/obs/config/sss_smap_jpl.yaml | 4 ++-- parm/soca/obs/config/sss_smos.yaml | 4 ++-- parm/soca/obs/config/sss_smos_esa.yaml | 4 ++-- parm/soca/obs/config/sst_gmi_l3u.yaml | 4 ++-- parm/soca/obs/config/sst_metopa_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_metopb_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_metopc_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_noaa18_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_noaa19_l3u.yaml | 4 ++-- parm/soca/obs/config/sst_noaa19_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml | 4 ++-- parm/soca/obs/config/sst_windsat_l3u.yaml | 4 ++-- 35 files changed, 68 insertions(+), 68 deletions(-) diff --git a/parm/soca/obs/config/adt_3a_egm2008.yaml b/parm/soca/obs/config/adt_3a_egm2008.yaml index c189d13be..2fa81e3fa 100644 --- a/parm/soca/obs/config/adt_3a_egm2008.yaml +++ b/parm/soca/obs/config/adt_3a_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_3a_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_3b_egm2008.yaml b/parm/soca/obs/config/adt_3b_egm2008.yaml index db99d259c..b3b475d70 100644 --- a/parm/soca/obs/config/adt_3b_egm2008.yaml +++ b/parm/soca/obs/config/adt_3b_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_3b_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_6a_egm2008.yaml b/parm/soca/obs/config/adt_6a_egm2008.yaml index c63ee80d2..5de4036ed 100644 --- a/parm/soca/obs/config/adt_6a_egm2008.yaml +++ b/parm/soca/obs/config/adt_6a_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_6a_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_all.yaml b/parm/soca/obs/config/adt_all.yaml index a9344848e..b7c6697dc 100644 --- a/parm/soca/obs/config/adt_all.yaml +++ b/parm/soca/obs/config/adt_all.yaml @@ -1,7 +1,5 @@ obs space: name: adt_all - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_c2_egm2008.yaml b/parm/soca/obs/config/adt_c2_egm2008.yaml index 34e748ca2..582ccb5c1 100644 --- a/parm/soca/obs/config/adt_c2_egm2008.yaml +++ b/parm/soca/obs/config/adt_c2_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_c2_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_coperl4.yaml b/parm/soca/obs/config/adt_coperl4.yaml index aed6c77d7..80229284c 100644 --- a/parm/soca/obs/config/adt_coperl4.yaml +++ b/parm/soca/obs/config/adt_coperl4.yaml @@ -1,7 +1,5 @@ obs space: name: adt_coperl4 - get values: - time interpolation: linear obsdatain: engine: type: H5File diff --git a/parm/soca/obs/config/adt_j2.yaml b/parm/soca/obs/config/adt_j2.yaml index 932421258..88177e49b 100644 --- a/parm/soca/obs/config/adt_j2.yaml +++ b/parm/soca/obs/config/adt_j2.yaml @@ -11,6 +11,8 @@ obs space: type: H5File obsfile: !ENV ${DATA}/diags/adt_j3.${PDY}${cyc}.nc4 simulated variables: [absoluteDynamicTopography] +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_j2_egm2008.yaml b/parm/soca/obs/config/adt_j2_egm2008.yaml index 8b58c5f2b..c24d85dab 100644 --- a/parm/soca/obs/config/adt_j2_egm2008.yaml +++ b/parm/soca/obs/config/adt_j2_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_j2_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_j3.yaml b/parm/soca/obs/config/adt_j3.yaml index bcefee767..d7ba552b9 100644 --- a/parm/soca/obs/config/adt_j3.yaml +++ b/parm/soca/obs/config/adt_j3.yaml @@ -1,7 +1,5 @@ obs space: name: adt_j3 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -11,6 +9,8 @@ obs space: type: H5File obsfile: !ENV ${DATA}/diags/adt_j3.${PDY}${cyc}.nc4 simulated variables: [absoluteDynamicTopography] +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_j3_egm2008.yaml b/parm/soca/obs/config/adt_j3_egm2008.yaml index cbd96970b..681072829 100644 --- a/parm/soca/obs/config/adt_j3_egm2008.yaml +++ b/parm/soca/obs/config/adt_j3_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_j3_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/adt_sa_egm2008.yaml b/parm/soca/obs/config/adt_sa_egm2008.yaml index ccd0cbbb8..3e564bffd 100644 --- a/parm/soca/obs/config/adt_sa_egm2008.yaml +++ b/parm/soca/obs/config/adt_sa_egm2008.yaml @@ -1,7 +1,5 @@ obs space: name: adt_sa_egm2008 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: ADT obs error: diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml index c70f68dd0..6365e7fb3 100644 --- a/parm/soca/obs/config/icec_amsr2_north.yaml +++ b/parm/soca/obs/config/icec_amsr2_north.yaml @@ -1,7 +1,5 @@ obs space: name: icec_amsr2_north - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml index 150549e9a..977644ef0 100644 --- a/parm/soca/obs/config/icec_amsr2_south.yaml +++ b/parm/soca/obs/config/icec_amsr2_south.yaml @@ -1,7 +1,5 @@ obs space: name: icec_amsr2_south - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_emc.yaml b/parm/soca/obs/config/icec_emc.yaml index 0d6ad6ae2..efa861d15 100644 --- a/parm/soca/obs/config/icec_emc.yaml +++ b/parm/soca/obs/config/icec_emc.yaml @@ -1,7 +1,5 @@ obs space: name: icec_emc - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -11,6 +9,8 @@ obs space: type: H5File obsfile: !ENV ${DATA}/diags/icec_emc.${PDY}${cyc}.nc4 simulated variables: [seaIceFraction] +get values: + time interpolation: linear obs operator: name: Identity observation alias file: ./obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_nsidc_nh.yaml b/parm/soca/obs/config/icec_nsidc_nh.yaml index 8581cd75a..6ce9984d1 100644 --- a/parm/soca/obs/config/icec_nsidc_nh.yaml +++ b/parm/soca/obs/config/icec_nsidc_nh.yaml @@ -1,7 +1,5 @@ obs space: name: icec_nsidc_nh - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_ice_area_fraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/icec_nsidc_sh.yaml b/parm/soca/obs/config/icec_nsidc_sh.yaml index da5d09a38..369964d15 100644 --- a/parm/soca/obs/config/icec_nsidc_sh.yaml +++ b/parm/soca/obs/config/icec_nsidc_sh.yaml @@ -1,7 +1,5 @@ obs space: name: icec_nsidc_sh - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_ice_area_fraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/icec_ssmis_f17_north.yaml b/parm/soca/obs/config/icec_ssmis_f17_north.yaml index 178ecce74..cbb8b3622 100644 --- a/parm/soca/obs/config/icec_ssmis_f17_north.yaml +++ b/parm/soca/obs/config/icec_ssmis_f17_north.yaml @@ -1,7 +1,5 @@ obs space: name: icec_ssmis_f17_north - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f17_south.yaml b/parm/soca/obs/config/icec_ssmis_f17_south.yaml index 5b26405b8..b7279f944 100644 --- a/parm/soca/obs/config/icec_ssmis_f17_south.yaml +++ b/parm/soca/obs/config/icec_ssmis_f17_south.yaml @@ -1,7 +1,5 @@ obs space: name: icec_ssmis_f17_south - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_north.yaml b/parm/soca/obs/config/icec_ssmis_f18_north.yaml index 75e71c15d..9e5676c93 100644 --- a/parm/soca/obs/config/icec_ssmis_f18_north.yaml +++ b/parm/soca/obs/config/icec_ssmis_f18_north.yaml @@ -1,7 +1,5 @@ obs space: name: icec_ssmis_f18_north - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_south.yaml b/parm/soca/obs/config/icec_ssmis_f18_south.yaml index 80e56a9a7..2d68b119a 100644 --- a/parm/soca/obs/config/icec_ssmis_f18_south.yaml +++ b/parm/soca/obs/config/icec_ssmis_f18_south.yaml @@ -1,7 +1,5 @@ obs space: name: icec_ssmis_f18_north - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaIceFraction] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/icefb_gdr.yaml b/parm/soca/obs/config/icefb_gdr.yaml index 4e26ef7e1..5554e395d 100644 --- a/parm/soca/obs/config/icefb_gdr.yaml +++ b/parm/soca/obs/config/icefb_gdr.yaml @@ -1,7 +1,5 @@ obs space: name: icefb_GDR - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_ice_freeboard] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: SeaIceThickness obs error: diff --git a/parm/soca/obs/config/sss_smap.yaml b/parm/soca/obs/config/sss_smap.yaml index 7efca37a4..e57f65d71 100644 --- a/parm/soca/obs/config/sss_smap.yaml +++ b/parm/soca/obs/config/sss_smap.yaml @@ -1,7 +1,5 @@ obs space: name: sss_smap - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -11,6 +9,8 @@ obs space: type: H5File obsfile: !ENV ${DATA}/diags/sss_smap.${PDY}${cyc}.nc4 simulated variables: [seaSurfaceSalinity] +get values: + time interpolation: linear obs operator: name: Identity observation alias file: ./obsop_name_map.yaml diff --git a/parm/soca/obs/config/sss_smap_jpl.yaml b/parm/soca/obs/config/sss_smap_jpl.yaml index c05d516df..cc77ce0af 100644 --- a/parm/soca/obs/config/sss_smap_jpl.yaml +++ b/parm/soca/obs/config/sss_smap_jpl.yaml @@ -1,7 +1,5 @@ obs space: name: sss_smap_jpl - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_surface_salinity] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/sss_smos.yaml b/parm/soca/obs/config/sss_smos.yaml index 7a61e3e97..2a2f973de 100644 --- a/parm/soca/obs/config/sss_smos.yaml +++ b/parm/soca/obs/config/sss_smos.yaml @@ -1,7 +1,5 @@ obs space: name: sss_smos - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_surface_salinity] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/sss_smos_esa.yaml b/parm/soca/obs/config/sss_smos_esa.yaml index d04cd4bf5..abfbca06e 100644 --- a/parm/soca/obs/config/sss_smos_esa.yaml +++ b/parm/soca/obs/config/sss_smos_esa.yaml @@ -1,7 +1,5 @@ obs space: name: sss_smos_esa - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_surface_salinity] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/sst_gmi_l3u.yaml b/parm/soca/obs/config/sst_gmi_l3u.yaml index e5985bd1a..d7cc3bb12 100644 --- a/parm/soca/obs/config/sst_gmi_l3u.yaml +++ b/parm/soca/obs/config/sst_gmi_l3u.yaml @@ -1,7 +1,5 @@ obs space: name: sst_gmi_l3u - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_surface_temperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml index dd93cc014..92859144a 100644 --- a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_metopa_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml index 865c6e472..8865929c1 100644 --- a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_metopb_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml index 6dd621e3e..1a4db76f6 100644 --- a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_metopc_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml index 593aa419e..7d0907bac 100644 --- a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_noaa18_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u.yaml b/parm/soca/obs/config/sst_noaa19_l3u.yaml index 28d9689e6..f8dfb79ba 100644 --- a/parm/soca/obs/config/sst_noaa19_l3u.yaml +++ b/parm/soca/obs/config/sst_noaa19_l3u.yaml @@ -1,7 +1,5 @@ obs space: name: sst_noaa19_l3u - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -11,6 +9,8 @@ obs space: type: H5File obsfile: !ENV ${DATA}/diags/sst_noaa19_l3u.${PDY}${cyc}.nc4 simulated variables: [seaSurfaceTemperature] +get values: + time interpolation: linear obs operator: name: Identity observation alias file: ./obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml index 2ef9b3f75..0f54bcb25 100644 --- a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_noaa19_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml index af3a157da..c0a75548a 100644 --- a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_viirs_n20_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml index a99ab6349..7f5987343 100644 --- a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml @@ -1,7 +1,5 @@ obs space: name: sst_viirs_npp_l3u_so025 - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity observation alias file: obsop_name_map.yaml diff --git a/parm/soca/obs/config/sst_windsat_l3u.yaml b/parm/soca/obs/config/sst_windsat_l3u.yaml index 1d320036c..b4cc4a28c 100644 --- a/parm/soca/obs/config/sst_windsat_l3u.yaml +++ b/parm/soca/obs/config/sst_windsat_l3u.yaml @@ -1,7 +1,5 @@ obs space: name: sst_windsat_l3u - get values: - time interpolation: linear obsdatain: engine: type: H5File @@ -13,6 +11,8 @@ obs space: simulated variables: [sea_surface_temperature] io pool: max pool size: 1 +get values: + time interpolation: linear obs operator: name: Identity obs error: From 22f905c943cde7035f11229c338d79cfdc23ddc0 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Thu, 30 Nov 2023 11:23:56 -0500 Subject: [PATCH 10/22] Add IODA conversions to marine obs prep task ex-script (#719) As a draft PR this is to allow some mid-development input. - `parm/soca/obsproc/obs_list_obsproc.yaml` is added as the `OBS_LIST` for testing purposes - `parm/soca/obsproc/obsproc_config.yaml` is added, containing the obs source-specific info in the yamls in `utils/test/testinput/` and the dictionary of observation source filename regexp patterns in `ush/soca/prep_marine_obs.py` - the yaml here is loaded to fetch the observation files and dynamically generate the yaml to feed to the IODA converter. I would be curious if the hierarchy or terminology breaks any established JEDI conventions. - `scripts/exglobal_prep_ocean_obs.py` now fetches the observation files to the RUNDIR, creates the IODA converter yaml with the input set to the filenames now returned by `prep_marine_obs.obs_fetch`, and runs the IODA converters. - some variable names have been changed to camelCase Also: this would require some rearranging of test data - right now I'm testing against the full files we used for IODA converters, but for the ctests I presume there would have to be some arrangement like with the IODA converter ctests, making NetCDF files out of text files. Lemme know what you think. --------- Co-authored-by: Guillaume Vernieres --- parm/soca/obsproc/obs_list_obsproc.yaml | 11 +++++ parm/soca/obsproc/obsproc_config.yaml | 25 +++++++++++ parm/soca/obsproc/obsproc_config_test.yaml | 25 +++++++++++ scripts/exglobal_prep_ocean_obs.py | 52 ++++++++++++++++++---- test/soca/gw/CMakeLists.txt | 14 ++++-- test/soca/gw/run_jjobs.yaml.test | 4 +- test/soca/gw/setup_obsproc.sh | 17 +++++++ ush/soca/prep_marine_obs.py | 27 +++++------ ush/soca/run_jjobs.py | 6 +++ utils/test/prepdata.sh | 4 ++ 10 files changed, 158 insertions(+), 27 deletions(-) create mode 100644 parm/soca/obsproc/obs_list_obsproc.yaml create mode 100644 parm/soca/obsproc/obsproc_config.yaml create mode 100644 parm/soca/obsproc/obsproc_config_test.yaml create mode 100755 test/soca/gw/setup_obsproc.sh diff --git a/parm/soca/obsproc/obs_list_obsproc.yaml b/parm/soca/obsproc/obs_list_obsproc.yaml new file mode 100644 index 000000000..187ac842a --- /dev/null +++ b/parm/soca/obsproc/obs_list_obsproc.yaml @@ -0,0 +1,11 @@ +observers: +- !INC ${OBS_YAML_DIR}/adt_all.yaml +- !INC ${OBS_YAML_DIR}/adt_j3.yaml +- !INC ${OBS_YAML_DIR}/adt_j2.yaml +#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml +- !INC ${OBS_YAML_DIR}/sss_smap.yaml +- !INC ${OBS_YAML_DIR}/sss_smos.yaml +- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml +- !INC ${OBS_YAML_DIR}/icec_emc.yaml +- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml diff --git a/parm/soca/obsproc/obsproc_config.yaml b/parm/soca/obsproc/obsproc_config.yaml new file mode 100644 index 000000000..60cb0df66 --- /dev/null +++ b/parm/soca/obsproc/obsproc_config.yaml @@ -0,0 +1,25 @@ +observations: +- obs space: + name: sss_smap + obsproc subdir: SSS + obsproc regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5 + provider: SMAP + output file: sss_smap.ioda.nc +- obs space: + name: sss_smos + provider: SMOS + obsproc subdir: SSS + output file: sss_smos.ioda.nc + obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc +- obs space: + name: adt_3b_egm2008 + obsproc subdir: ADT + obsproc regex: rads_adt_3b_???????.nc + provider: RADS + output file: adt_all.nc4 +- obs space: + name: icec_amsr2_north + provider: AMSR2 + obsproc subdir: icec + output file: icec_amsr2_north_1.ioda.nc + obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc diff --git a/parm/soca/obsproc/obsproc_config_test.yaml b/parm/soca/obsproc/obsproc_config_test.yaml new file mode 100644 index 000000000..0860428cb --- /dev/null +++ b/parm/soca/obsproc/obsproc_config_test.yaml @@ -0,0 +1,25 @@ +observations: +- obs space: + name: sss_smap + obsproc subdir: '' + obsproc regex: sss_smap_?.nc4 + provider: SMAP + output file: sss_smap.ioda.nc +- obs space: + name: sss_smos + provider: SMOS + obsproc subdir: '' + output file: sss_smos.ioda.nc + obsproc regex: sss_smos_?.nc4 +- obs space: + name: adt_3b_egm2008 + obsproc subdir: '' + obsproc regex: rads_adt_??_???????.nc4 + provider: RADS + output file: adt_all.nc4 +- obs space: + name: icec_amsr2_north + provider: AMSR2 + obsproc subdir: '' + output file: icec_amsr2_north.ioda.nc + obsproc regex: icec_amsr2_north_?.nc4 diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index da12d2b7b..ff424557a 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -1,17 +1,53 @@ #!/usr/bin/env python3 # exglobal_prep_ocean_obs.py # Pepares observations for marine DA +from datetime import datetime, timedelta +import logging import os -from wxflow import YAMLFile import prep_marine_obs -import logging +import subprocess +from wxflow import YAMLFile, save_as_yaml + +# set up logger +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') + +cyc = os.getenv('cyc') +PDY = os.getenv('PDY') + +# set the window times +cdateDatetime = datetime.strptime(PDY + cyc, '%Y%m%d%H') +windowBeginDatetime = cdateDatetime - timedelta(hours=3) +windowEndDatetime = cdateDatetime + timedelta(hours=3) +windowBegin = windowBeginDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') +windowEnd = windowEndDatetime.strftime('%Y-%m-%dT%H:%M:%SZ') + +OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC') OBS_YAML = os.getenv('OBS_YAML') +obsConfig = YAMLFile(OBS_YAML) + +OBSPROC_YAML = os.getenv('OBSPROC_YAML') +obsprocConfig = YAMLFile(OBSPROC_YAML) + +# TODO (AFE): needs more error handling (missing sources, missing files) +for observer in obsConfig['observers']: + + obsSpaceName = observer['obs space']['name'] + print(f"obsSpaceName: {obsSpaceName}") + + for observation in obsprocConfig['observations']: + + obsprocSpace = observation['obs space'] + obsprocSpaceName = obsprocSpace['name'] + + if obsprocSpaceName == obsSpaceName: + + matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace) + obsprocSpace['input files'] = matchingFiles + obsprocSpace['window begin'] = windowBegin + obsprocSpace['window end'] = windowEnd -data = YAMLFile(OBS_YAML) -print(data) + iodaYamlFilename = obsprocSpaceName + '2ioda.yaml' + save_as_yaml(obsprocSpace, iodaYamlFilename) -for observer in data['observers']: - obs_source_name = observer['obs space']['name'] - logging.info(f"obs_source_name: {obs_source_name}") - prep_marine_obs.obs_fetch(obs_source_name) + subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True) diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index c5a3a26ca..b1f92b65f 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -36,14 +36,22 @@ add_test(NAME test_gdasapp_soca_run_clean # Create scratch for testing apps file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch) +# Create scratch for obs prep task +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) + +# Set up obsproc, based on test_gdasapp_util_prepdata +add_test(NAME test_gdasapp_soca_setup_obsproc + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) + # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_* -set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" +set(jjob_list "JGLOBAL_PREP_OCEAN_OBS" + "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP" "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT" "JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN" "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT" "JGDAS_GLOBAL_OCEAN_ANALYSIS_POST" - "JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY" - "JGLOBAL_PREP_OCEAN_OBS") + "JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY") set(setup "") foreach(jjob ${jjob_list}) diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 3270d820a..683ad9135 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -57,7 +57,9 @@ setup_expt config: NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 prepoceanobs: - SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obs_list_obsproc.yaml + OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config_test.yaml + DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/obsproc job options: account: da-cpu diff --git a/test/soca/gw/setup_obsproc.sh b/test/soca/gw/setup_obsproc.sh new file mode 100755 index 000000000..bb4eae4c3 --- /dev/null +++ b/test/soca/gw/setup_obsproc.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -ex + +project_source_dir=$1 + +# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command +test_dmpdir="gdas.20180415/12" + +rm -rf ${test_dmpdir} +mkdir -p ${test_dmpdir} + +cd ${test_dmpdir} + +${project_source_dir}/utils/test/prepdata.sh ${project_source_dir}/utils/test/ + + + diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py index a0d137d2b..7202d6be9 100755 --- a/ush/soca/prep_marine_obs.py +++ b/ush/soca/prep_marine_obs.py @@ -63,32 +63,29 @@ } -def obs_fetch(obs_source_name): +def obs_fetch(obsprocSpace): - try: - obs_source = obs_dict[obs_source_name] - except KeyError: - print(f'WARNING: no obs source {obs_source_name} defined, skipping') - return - - subdir = obs_source[0] - filepattern = obs_source[1] + subdir = obsprocSpace['obsproc subdir'] + filepattern = obsprocSpace['obsproc regex'] datadir = os.path.join(cycdir, subdir) # TODO: check the existence of this print('datadir:', datadir) - matching_files = [] + matchingFiles = [] for root, _, files in os.walk(datadir): for filename in fnmatch.filter(files, filepattern): - matching_files.append((root, filename)) + matchingFiles.append(filename) obs_cpy = [] - for obs_src in matching_files: - obs_path = os.path.join(obs_src[0], obs_src[1]) - obs_dst = os.path.join(COMIN_OBS, obs_src[1]) + for obs_src in matchingFiles: + obs_path = os.path.join(datadir, obs_src) + obs_dst = os.path.join(COMIN_OBS, obs_src) obs_cpy.append([obs_path, obs_dst]) - print(obs_cpy) + print(f"obs_cpy: {obs_cpy}") + print(f"matchingFiles: {matchingFiles}") FileHandler({'copy': obs_cpy}).sync() + + return matchingFiles diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index 80d8a3c54..4e0212da2 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -217,6 +217,12 @@ def fixconfigs(self): 'STMP': self.stmp, 'ROTDIR': self.rotdir, 'EXPDIRS': self.expdirs} + + # needed for this ctest at least until R2D2 goes away + if 'JGLOBAL_PREP_OCEAN_OBS' in self.config['jjobs']: + dmpdir = self.config['setup_expt config']['prepoceanobs']['DMPDIR'] + var2replace['DMPDIR'] = dmpdir + with open(configbase, 'r') as f: newconfigbase = f.read() for key, value in var2replace.items(): diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh index 4c47f4de9..f6a1c0698 100755 --- a/utils/test/prepdata.sh +++ b/utils/test/prepdata.sh @@ -1,4 +1,8 @@ #!/bin/bash +# called for test_gdasapp_util_prepdata, and by +# test/soca/gw/setup_obsproc.sh for test_gdasapp_soca_setup_obsproc + + set -e cdl2nc4() { From da5595f4240d6e5323d7c73b79b7796ee633a64d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 4 Dec 2023 08:58:53 -0500 Subject: [PATCH 11/22] Recentering around the deterministic (#787) ### Work done - Minor modification of `gdas_ens_handler.h` to allow generating increments to recenter the ensemble forecast around the deterministic. - commented out the 2 broken ioda converter tests - update to atlas 0.35 for those of us who are building altlas --- CMakeLists.txt | 2 +- parm/soca/berror/soca_ensb.yaml | 4 ++- utils/soca/gdas_ens_handler.h | 58 +++++++++++++++++++++------------ utils/test/CMakeLists.txt | 22 +++++++------ 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b26b64b5b..2091893fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,7 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) - ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.29.0 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) # External (required) observation operators option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF diff --git a/parm/soca/berror/soca_ensb.yaml b/parm/soca/berror/soca_ensb.yaml index 42e054531..6041ab8a7 100644 --- a/parm/soca/berror/soca_ensb.yaml +++ b/parm/soca/berror/soca_ensb.yaml @@ -16,7 +16,9 @@ vertical geometry: ocn_filename: MOM.res.nc read_from_file: 3 -soca increments: +add recentering increment: false + +soca increments: # Could also be states, but they are read as increments number of increments: ${ENS_SIZE} pattern: '%mem%' template: diff --git a/utils/soca/gdas_ens_handler.h b/utils/soca/gdas_ens_handler.h index 5cf862651..315039bbe 100644 --- a/utils/soca/gdas_ens_handler.h +++ b/utils/soca/gdas_ens_handler.h @@ -97,6 +97,12 @@ namespace gdasapp { ensMembers.push_back(postProcIncr.read(i)); } + // Check if we need to recenter the increment around the deterministic + bool addRecenterIncr(false); + if ( fullConfig.has("add recentering increment") ) { + fullConfig.get("add recentering increment", addRecenterIncr); + } + // Compute ensemble moments soca::Increment ensMean(geom, postProcIncr.socaIncrVar_, postProcIncr.dt_); soca::Increment ensStd(geom, postProcIncr.socaIncrVar_, postProcIncr.dt_); @@ -120,24 +126,25 @@ namespace gdasapp { fullConfig.get("steric height", stericVarChangeConfig); oops::Log::info() << "steric config 0000: " << stericVarChangeConfig << std::endl; - // Initialize trajectories + // Initialize the trajectories used in the linear variable changes const eckit::LocalConfiguration trajConfig(fullConfig, "trajectory"); - soca::State cycleTraj(geom, trajConfig); // trajectory of the cycle - soca::State meanTraj(cycleTraj); // trajectory defined as the ens. mean - meanTraj.zero(); - meanTraj += ensMean; - - // Compute the error between the ensemble mean and the deterministic - soca::Increment deterministicError(geom, postProcIncr.socaIncrVar_, postProcIncr.dt_); - deterministicError.diff(cycleTraj, meanTraj); + soca::State determTraj(geom, trajConfig); // deterministic trajectory + soca::State ensMeanTraj(determTraj); // trajectory defined as the ens. mean + ensMeanTraj.zero(); + ensMeanTraj += ensMean; + + // Compute the recentering increment as the difference between + // the ensemble mean and the deterministic + soca::Increment recenteringIncr(geom, postProcIncr.socaIncrVar_, postProcIncr.dt_); + recenteringIncr.diff(determTraj, ensMeanTraj); eckit::LocalConfiguration sshRecErrOutputConfig(fullConfig, "ssh output.recentering error"); - deterministicError = postProcIncr.setToZero(deterministicError); + recenteringIncr = postProcIncr.setToZero(recenteringIncr); oops::Log::info() << "steric config : " << stericVarChangeConfig << std::endl; - postProcIncr.applyLinVarChange(deterministicError, stericVarChangeConfig, cycleTraj); - oops::Log::info() << "ensemble mean: " << meanTraj << std::endl; - oops::Log::info() << "deterministic: " << cycleTraj << std::endl; - oops::Log::info() << "error: " << deterministicError << std::endl; - deterministicError.write(sshRecErrOutputConfig); + postProcIncr.applyLinVarChange(recenteringIncr, stericVarChangeConfig, determTraj); + oops::Log::info() << "ensemble mean: " << ensMeanTraj << std::endl; + oops::Log::info() << "deterministic: " << determTraj << std::endl; + oops::Log::info() << "error: " << recenteringIncr << std::endl; + recenteringIncr.write(sshRecErrOutputConfig); // Re-process the ensemble of perturbations int result = 0; @@ -165,7 +172,7 @@ namespace gdasapp { // Compute the original steric height perturbation from T and S eckit::LocalConfiguration stericConfig(fullConfig, "steric height"); - postProcIncr.applyLinVarChange(incr, stericConfig, meanTraj); + postProcIncr.applyLinVarChange(incr, stericConfig, ensMeanTraj); ssh_tmp = incr; sshSteric.push_back(ssh_tmp); @@ -183,10 +190,10 @@ namespace gdasapp { incr = postProcIncr.setToZero(incr); // Filter ensemble member and recompute steric ssh, recentering around - // the cycle's trajectory + // the deterministic trajectory if ( fullConfig.has("linear variable change") ) { eckit::LocalConfiguration lvcConfig(fullConfig, "linear variable change"); - postProcIncr.applyLinVarChange(incr, lvcConfig, cycleTraj); + postProcIncr.applyLinVarChange(incr, lvcConfig, determTraj); } // Add the unbalanced ssh to the recentered perturbation @@ -200,12 +207,23 @@ namespace gdasapp { incr.fromFieldSet(incrFs); oops::Log::debug() << "&&&&& after adding ssh_u " << incr << std::endl; - // Save final perturbation, used in the offline EnVAR + // Save final perturbation, used in the EnVAR or to initialize the ensemble forecast + if (addRecenterIncr) { + // Add the recentering increment to the increment ensemble members. + // This is generally used to recenter the ensemble fcst around the + // deterministic + incr += recenteringIncr; + } result = postProcIncr.save(incr, i+1); // Update ensemble ensMembers[i] = incr; - } + } + + // Exit early if only recentering around the deterministic + if (addRecenterIncr) { + return 0; + } // Compute ensemble moments for total ssh soca::Increment sshMean(geom, socaSshVar, postProcIncr.dt_); diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index 96aff4631..a53c92c2e 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -62,11 +62,12 @@ ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the SMAP to IODA converter -ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda - COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x - ARGS "../testinput/gdas_smap2ioda.yaml" - LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +# TODO(Mindo): Turn back on when date is fixed +#ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda +# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x +# ARGS "../testinput/gdas_smap2ioda.yaml" +# LIBS gdas-utils +# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the SMOS to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda @@ -76,8 +77,9 @@ ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the AMSR2 to IODA converter -ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda - COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x - ARGS "../testinput/gdas_icecamsr2ioda.yaml" - LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +# TODO(Mindo): Turn back on when date is fixed +#ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda +# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x +# ARGS "../testinput/gdas_icecamsr2ioda.yaml" +# LIBS gdas-utils +# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) From 50f5ebcab51c05e64a5bc4f3f5a571f628cf0677 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 4 Dec 2023 10:53:31 -0500 Subject: [PATCH 12/22] Remove the bump correlation configuration (#791) Mostly just tidying the code, we have not made use of the BUMP correlation in a while and I don't think we will anytime soon. It's still used for localization. --- scripts/exgdas_global_marine_analysis_bmat.sh | 104 ++---------------- scripts/exgdas_global_marine_analysis_prep.py | 62 +---------- 2 files changed, 13 insertions(+), 153 deletions(-) diff --git a/scripts/exgdas_global_marine_analysis_bmat.sh b/scripts/exgdas_global_marine_analysis_bmat.sh index 5cd52272d..6afaa27e9 100755 --- a/scripts/exgdas_global_marine_analysis_bmat.sh +++ b/scripts/exgdas_global_marine_analysis_bmat.sh @@ -36,36 +36,6 @@ pwd=$(pwd) # Utilities export NLN=${NLN:-"/bin/ln -sf"} - -function bump_vars() -{ - tmp=$(ls -d ${1}_* ) - lov=() - for v in $tmp; do - lov[${#lov[@]}]=${v#*_} - done - echo "$lov" -} - -function concatenate_bump() -{ - bumpdim=$1 - # Concatenate the bump files - vars=$(bump_vars $bumpdim) - n=$(wc -w <<< "$vars") - echo "concatenating $n variables: $vars" - lof=$(ls ${bumpdim}_${vars[0]}) - echo $lof - for f in $lof; do - bumpbase=${f#*_} - output=bump/${bumpdim}_$bumpbase - lob=$(ls ${bumpdim}_*/*$bumpbase) - for b in $lob; do - ncks -A $b $output - done - done -} - function clean_yaml() { mv $1 tmp_yaml; @@ -86,23 +56,6 @@ else fi fi -################################################################################ -# Prepare the diagonal of the parametric B -shopt -s nullglob -files=(ocn.bkgerr_stddev.incr.*.nc) -echo $files -if [ ${#files[@]} -gt 0 ]; then - echo "Diag of B already staged, skipping the parametric diag B" -else - # TODO: this step should be replaced by a post processing step of the ens. derived std. dev. - $APRUN_OCNANAL $JEDI_BIN/soca_convertincrement.x parametric_stddev_b.yaml > parametric_stddev_b.out 2>&1 - export err=$?; err_chk - if [ $err -gt 0 ]; then - exit $err - fi -fi -shopt -u nullglob - ################################################################################ # Write ensemble weights for the hybrid envar $APRUN_OCNANAL $JEDI_BIN/gdas_socahybridweights.x soca_ensweights.yaml @@ -112,16 +65,18 @@ if [ $err -gt 0 ]; then fi ################################################################################ -# Compute the ens std. dev, ignore ssh variance -# TODO (G): Implement what's below into one single oops application -# 0 - Compute moments of original ensemble, used at the diag of the first -# component of the hybrid B -# 1 - Ensemble perturbations: -# o Vertically Interpolate to the deterministic layers -# o Recenter around 0 to create an ensemble of perurbations -# 2 - Filter members and apply the linear steric height balance to each members -# 3 - Copy h from deterministic to unbalanced perturbations -# 4 - Compute moments of converted ensemble perturbations +# Ensemble perturbations for the EnVAR and diagonal of static B +# Static B: +# - Compute moments of original ensemble with the balanced ssh removed +# from the statistics +# +# Ensemble of perturbations: +# - apply the linear steric height balance to each members, using the +# deterministic for trajectory. +# - add the unblanced ssh to the steric ssh field +# Diagnostics: +# - variance explained by the steric heigh +# - moments # Process static ensemble clean_yaml soca_clim_ens_moments.yaml @@ -146,14 +101,6 @@ if [ ${#files[@]} -gt 0 ]; then shopt -u nullglob fi -################################################################################ -# Set decorrelation scales for bump C -$APRUN_OCNANAL $JEDI_BIN/soca_setcorscales.x soca_setcorscales.yaml -export err=$?; err_chk -if [ $err -gt 0 ]; then - exit $err -fi - ################################################################################ # Set localization scales for the hybrid en. var. $APRUN_OCNANAL $JEDI_BIN/soca_setcorscales.x soca_setlocscales.yaml @@ -162,33 +109,6 @@ if [ $err -gt 0 ]; then exit $err fi -################################################################################ -# Compute convolution coefs for C -# TODO (G, C, R, ...): problem with ' character when reading yaml, removing from file for now -# 2D C from bump -if false; then - # TODO: resurect this section when making use of bump 3D in the static B, skip for now - yaml_bump2d=soca_bump2d.yaml - clean_yaml $yaml_bump2d - $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_toolbox.x $yaml_bump2d 2>$yaml_bump2d.err - export err=$?; err_chk - if [ $err -gt 0 ]; then - exit $err - fi - - # 3D C from bump - yaml_list=`ls soca_bump3d_*.yaml` - for yaml in $yaml_list; do - clean_yaml $yaml - $APRUN_OCNANAL $JEDI_BIN/soca_error_covariance_toolbox.x $yaml 2>$yaml.err - export err=$?; err_chk - if [ $err -gt 0 ]; then - exit $err - fi - done - concatenate_bump 'bump3d' -fi - ################################################################################ # Compute convolution coefs for L clean_yaml soca_bump_loc.yaml diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index aaeadc612..b5e7117fc 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -308,16 +308,6 @@ def find_clim_ens(input_date): logging.info(f"---------------- Stage static files") ufsda.stage.soca_fix(stage_cfg) -################################################################################ -# stage background error files - -logging.info(f"---------------- Stage static files") -bkgerr_list = [] -for domain in ['ocn', 'ice']: - fname_stddev = find_bkgerr(pytz.utc.localize(window_begin, is_dst=None), domain=domain) - fname_out = domain+'.bkgerr_stddev.incr.'+window_begin_iso+'.nc' - bkgerr_list.append([fname_stddev, fname_out]) -FileHandler({'copy': bkgerr_list}).sync() ################################################################################ # stage ensemble members @@ -389,14 +379,6 @@ def find_clim_ens(input_date): config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) config.save(berr_yaml) -################################################################################ -# copy yaml for decorrelation length scales - -logging.info(f"---------------- generate soca_setcorscales.yaml") -corscales_yaml_src = os.path.join(gdas_home, 'parm', 'soca', 'berror', 'soca_setcorscales.yaml') -corscales_yaml_dst = os.path.join(stage_cfg['stage_dir'], 'soca_setcorscales.yaml') -FileHandler({'copy': [[corscales_yaml_src, corscales_yaml_dst]]}).sync() - ################################################################################ # copy yaml for localization length scales @@ -408,49 +390,7 @@ def find_clim_ens(input_date): ################################################################################ # generate yaml for bump/nicas (used for correlation and/or localization) -logging.info(f"---------------- generate BUMP/NICAS yamls") -# TODO (Guillaume): move the possible vars somewhere else -vars3d = ['tocn', 'socn', 'uocn', 'vocn', 'chl', 'biop'] -vars2d = ['ssh', 'cicen', 'hicen', 'hsnon', 'swh', - 'sw', 'lw', 'lw_rad', 'lhf', 'shf', 'us'] - -# 2d bump yaml (all 2d vars at once) -bumpdir = 'bump' -ufsda.disk_utils.mkdir(os.path.join(anl_dir, bumpdir)) -bump_yaml = os.path.join(anl_dir, 'soca_bump2d.yaml') -bump_yaml_template = os.path.join(gdas_home, - 'parm', - 'soca', - 'berror', - 'soca_bump2d.yaml') -config = YAMLFile(path=bump_yaml_template) -config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) -config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) -config.save(bump_yaml) - -# 3d bump yaml, 1 yaml per variable -soca_vars = ['tocn', 'socn', 'uocn', 'vocn'] -for v in soca_vars: - logging.info(f"creating the yaml to initialize bump for {v}") - if v in vars2d: - continue - else: - dim = '3d' - bump_yaml = os.path.join(anl_dir, 'soca_bump'+dim+'_'+v+'.yaml') - bump_yaml_template = os.path.join(gdas_home, - 'parm', - 'soca', - 'berror', - 'soca_bump_split.yaml') - bumpdir = 'bump'+dim+'_'+v - os.environ['BUMPDIR'] = bumpdir - ufsda.disk_utils.mkdir(os.path.join(anl_dir, bumpdir)) - os.environ['CVAR'] = v - config = YAMLFile(path=bump_yaml_template) - config = Template.substitute_structure(config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) - config = Template.substitute_structure(config, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) - config.save(bump_yaml) - +logging.info(f"---------------- generate BUMP/NICAS localization yamls") # localization bump yaml bumpdir = 'bump' ufsda.disk_utils.mkdir(os.path.join(anl_dir, bumpdir)) From 654dcdf11f1ecb31cbb519fc2c90f2e8f4c172cf Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:13:43 -0500 Subject: [PATCH 13/22] Minor observation YAML updates for ATMS, SATWIND and SCATWIND (#789) This PR includes the following: 1. YAML name change - add sensor (ABI) to the filename of observation YAML for SATWIND from GOES platforms 2. YAML name change - add sensor (ASCAT) to the filename of observation YAML for SCATWIND from MetOp platforms 3. update the YAML name changes in the list for 3dvar 4. Add `CO2` in obs forward operator and add `linear obs operator` without CO2 for ATMS These changes have no impact on 3DVar results. --- parm/atm/obs/config/atms_n20.yaml | 4 +++- parm/atm/obs/config/atms_npp.yaml | 4 +++- .../{satwind_goes-16.yaml => satwind_abi_goes-16.yaml} | 0 .../{satwind_goes-17.yaml => satwind_abi_goes-17.yaml} | 0 ...atwind_metop-a.yaml => scatwind_ascat_metop-a.yaml} | 0 ...atwind_metop-b.yaml => scatwind_ascat_metop-b.yaml} | 0 parm/atm/obs/lists/gdas_prototype_3d.yaml | 10 +++++----- 7 files changed, 11 insertions(+), 7 deletions(-) rename parm/atm/obs/config/{satwind_goes-16.yaml => satwind_abi_goes-16.yaml} (100%) rename parm/atm/obs/config/{satwind_goes-17.yaml => satwind_abi_goes-17.yaml} (100%) rename parm/atm/obs/config/{scatwind_metop-a.yaml => scatwind_ascat_metop-a.yaml} (100%) rename parm/atm/obs/config/{scatwind_metop-b.yaml => scatwind_ascat_metop-b.yaml} (100%) diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 2a29f5d95..5cb9c27f3 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -15,7 +15,7 @@ obs space: obs operator: name: CRTM - Absorbers: [H2O,O3] + Absorbers: [H2O, O3, CO2] Clouds: [Water, Ice] Cloud_Fraction: 1.0 Cloud_Seeding: true @@ -23,6 +23,8 @@ obs operator: Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml index ecddd346c..0a9b633b6 100644 --- a/parm/atm/obs/config/atms_npp.yaml +++ b/parm/atm/obs/config/atms_npp.yaml @@ -15,7 +15,7 @@ obs space: obs operator: name: CRTM - Absorbers: [H2O,O3] + Absorbers: [H2O,O3,CO2] Clouds: [Water, Ice] Cloud_Fraction: 1.0 Cloud_Seeding: true @@ -23,6 +23,8 @@ obs operator: Sensor_ID: &Sensor_ID atms_npp EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias.nc4 diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_abi_goes-16.yaml similarity index 100% rename from parm/atm/obs/config/satwind_goes-16.yaml rename to parm/atm/obs/config/satwind_abi_goes-16.yaml diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_abi_goes-17.yaml similarity index 100% rename from parm/atm/obs/config/satwind_goes-17.yaml rename to parm/atm/obs/config/satwind_abi_goes-17.yaml diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_ascat_metop-a.yaml similarity index 100% rename from parm/atm/obs/config/scatwind_metop-a.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-a.yaml diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_ascat_metop-b.yaml similarity index 100% rename from parm/atm/obs/config/scatwind_metop-b.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-b.yaml diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index e8f2d006c..1c4d3a95c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -4,10 +4,10 @@ observers: ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/atms_npp.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -- !INC ${OBS_YAML_DIR}/satwind_goes-16.yaml -##- !INC ${OBS_YAML_DIR}/satwind_goes-17.yaml -##- !INC ${OBS_YAML_DIR}/scatwind_metop-a.yaml -##- !INC ${OBS_YAML_DIR}/scatwind_metop-b.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-16.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-17.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml @@ -15,4 +15,4 @@ observers: ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml -##- !INC ${OBS_YAML_DIR}/gnssro.yaml \ No newline at end of file +##- !INC ${OBS_YAML_DIR}/gnssro.yaml From 30ab4f928846b5a97235367fb76f51be038abf07 Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:48:02 -0500 Subject: [PATCH 14/22] Update IASI YAML files for end-to-end testing (#769) --- parm/atm/obs/config/iasi_metop-a.yaml | 756 ++++++++++++++++++++ parm/atm/obs/config/iasi_metop-b.yaml | 756 ++++++++++++++++++++ parm/atm/obs/lists/gdas_prototype_3d.yaml | 2 + parm/atm/obs/testing/iasi_metop-a.yaml | 22 +- parm/atm/obs/testing/iasi_metop-a_noqc.yaml | 158 ++++ parm/atm/obs/testing/iasi_metop-b.yaml | 22 +- parm/atm/obs/testing/iasi_metop-b_noqc.yaml | 158 ++++ parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml | 238 ++++++ 8 files changed, 2098 insertions(+), 14 deletions(-) create mode 100644 parm/atm/obs/config/iasi_metop-a.yaml create mode 100644 parm/atm/obs/config/iasi_metop-b.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-a_noqc.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-b_noqc.yaml create mode 100755 parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml diff --git a/parm/atm/obs/config/iasi_metop-a.yaml b/parm/atm/obs/config/iasi_metop-a.yaml new file mode 100644 index 000000000..f5c7a602a --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-a.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-a.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-a_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-a.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-a_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-a_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-a_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-a_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-a_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: iasi_metop-a + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-a_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-a + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-a_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/config/iasi_metop-b.yaml b/parm/atm/obs/config/iasi_metop-b.yaml new file mode 100644 index 000000000..1a4681dc6 --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-b.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-b.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-b_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-b.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-b_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-b_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-b_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-b_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-b_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + sensor: iasi_metop-b + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-b_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-b + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-b_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 1c4d3a95c..b32d4b634 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -13,6 +13,8 @@ observers: ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml ##- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/atm/obs/testing/iasi_metop-a.yaml b/parm/atm/obs/testing/iasi_metop-a.yaml index fed8883c4..477c1c871 100644 --- a/parm/atm/obs/testing/iasi_metop-a.yaml +++ b/parm/atm/obs/testing/iasi_metop-a.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-a_channels sensor: iasi_metop-a + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-a + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-a_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 127348 - +passedBenchmark: 144704 #127348 diff --git a/parm/atm/obs/testing/iasi_metop-a_noqc.yaml b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml new file mode 100644 index 000000000..c9ee251a4 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-a_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-a_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlap iasi_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 801416 diff --git a/parm/atm/obs/testing/iasi_metop-b.yaml b/parm/atm/obs/testing/iasi_metop-b.yaml index 2d0654822..660aae464 100644 --- a/parm/atm/obs/testing/iasi_metop-b.yaml +++ b/parm/atm/obs/testing/iasi_metop-b.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-b_channels sensor: iasi_metop-b + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-b + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-b_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 124731 - +passedBenchmark: 142616 # 124731 diff --git a/parm/atm/obs/testing/iasi_metop-b_noqc.yaml b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml new file mode 100644 index 000000000..700c46cd2 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-b_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-b_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlap iasi_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 790328 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml new file mode 100755 index 000000000..c15df2b62 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml @@ -0,0 +1,238 @@ +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.mtiasi.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteId: + query: "*/SAID" + + sensorId: + query: "*/SIID[1]" + + scanLineNumber: + query: "*/SLNM" + + gqisFlagQual: + query: "*/QGFQ" + + fieldOfViewNumber: + query: "*/FOVN" + + sensorScanPosition: + sensorScanPosition: + fieldOfViewNumber: "*/FOVN" + sensor: iasi + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + stationElevation: + query: "*/SELV" + type: float + + sensorViewAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.330 + scanStep: 3.334 + scanStepAdjust: 0.625 + sensor: iasi + + fractionOfClearPixelsInFOV: + query: "*/IASIL1CS{1}/FCPH" +# type: float +# transforms: +# - scale: 0.01 + + sensorChannelNumber: + query: "*/IASICHN/CHNM" + + # ObsValue + spectralRadiance: + spectralRadiance: + sensorChannelNumber: "*/IASICHN/CHNM" + startChannel: "*/IASIL1CB/STCH" + endChannel: "*/IASIL1CB/ENCH" + scaleFactor: "*/IASIL1CB/CHSF" + scaledSpectralRadiance: "*/IASICHN/SCRA" + + splits: + satId: + category: + variable: satelliteId + map: + _3: metop-b + _4: metop-a + _5: metop-c + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/IASICHN" + + - name: Cluster + path: "*/IASIL1CS" + + - name: Band + path: "*/IASIL1CB" + + globals: + - name: "platformCommonName" + type: string + value: "Meteorological Operational Satellite" + + - name: "platformLongDescription" + type: string + value: "EUMETSAT Polar System in sunsynchronous orbit" + + - name: "source" + type: string + value: "MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP)" + + - name: "sourceFiles" + type: string + value: "gdas.t00z.mtiasi.tm00.bufr_d" + + - name: "datetimeReference" + type: string + value: "2021-08-01T00:00:00Z" + + - name: "sensor" + type: string + value: "IASI" + + - name: "processingLevel" + type: string + value: "Level-1C" + + - name: "converter" + type: string + value: "BUFR" + + variables: + + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [ -90, 90 ] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [ -180, 180 ] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteId + longName: "Satellite Identifier" + + - name: "MetaData/instrumentIdentifier" + source: variables/sensorId + longName: "Satellite Instrument Identifier" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/qualityFlags" + source: variables/gqisFlagQual + longName: "Individual IASI-System Quality Flag" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/sensorScanPosition" + source: variables/sensorScanPosition + longName: "Sensor Scan Position" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [ 0, 180 ] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [ 0, 90 ] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorViewAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/stationElevation" + source: variables/stationElevation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/fractionOfClearPixelsInFOV" + source: variables/fractionOfClearPixelsInFOV + longName: "Fraction of Clear Pixels in a Field of View" + units: "1" + range: [0, 100] + +# The unit from BUFR is W m-2 sr-1 m -- this is radiance per wavenumber +# - name: "ObsValue/spectralRadiance" + - name: "ObsValue/radiance" + source: variables/spectralRadiance + longName: "IASI Spectral Radiance" + units: "W m-2 sr-1" +# units: "W m-2 sr-1 m-1" + From 7b379b27436060a140e17a6c2f773a45e34eafc6 Mon Sep 17 00:00:00 2001 From: PraveenKumar-NOAA <63739712+PraveenKumar-NOAA@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:48:50 -0500 Subject: [PATCH 15/22] ADPUPA prepbufr python/json updates (#793) A number of changes were made to the adpupa python script and json file: - TPC flags are changed for the airTemperature and virtualTemperature - Group and type are changed for the height variable - Variable verticalSignificance is changed to the prepbufrDataLevelCategory - Updated input/output file names in python script and some information in json file **Variable naming convention results:** Variable MetaData/prepbufrDataLevelCategory ERROR: Variable 'MetaData/prepbufrDataLevelCategory' is not listed in the YAML conventions file. Variable QCFlags/qualityFlags Variable ObsValue/stationPressure Variable ObsValue/airTemperature Variable ObsValue/virtualTemperature Variable ObsValue/specificHumidity Variable ObsValue/windEastward Variable ObsValue/windNorthward Variable QualityMarker/pressure Variable QualityMarker/stationPressure Variable QualityMarker/airTemperature Variable QualityMarker/windEastward Variable QualityMarker/virtualTemperature Variable QualityMarker/specificHumidity Variable QualityMarker/windNorthward **Final results:** _errors: 1_ _warnings: 0_ A new variable MetaData/prepbufrDataLevelCategory will be added to the YAML conventions file. --------- Co-authored-by: PraveenKumar-NOAA --- .../bufr2ioda/bufr2ioda_adpupa_prepbufr.json | 18 ++-- .../bufr2ioda/bufr2ioda_adpupa_prepbufr.py | 102 +++++++++--------- 2 files changed, 59 insertions(+), 61 deletions(-) diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json index f2156e0bf..e3905f397 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json +++ b/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json @@ -1,11 +1,11 @@ { - "data_format": "prepbufr", - "data_type": "prepbufr", - "cycle_type": "{{ RUN }}", - "cycle_datetime" : "{{ current_cycle | to_YMDH }}", - "dump_directory": "{{ DMPDIR }}", - "ioda_directory": "{{ COM_OBS }}", - "subsets": [ "ADPUPA" ], - "data_provider": "U.S. NOAA", - "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS", + "data_format" : "prepbufr", + "data_type" : "ADPUPA", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "ADPUPA" ], + "data_provider" : "U.S. NOAA", + "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS" } diff --git a/ush/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.py b/ush/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.py index 8e6640105..40a5bc9b9 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.py @@ -35,9 +35,9 @@ def Compute_dateTime(cycleTimeSinceEpoch, hrdr): hrdr = np.int64(hrdr*3600) - hrdr += cycleTimeSinceEpoch + dateTime = hrdr + cycleTimeSinceEpoch - return hrdr + return dateTime def bufr_to_ioda(config, logger): @@ -46,7 +46,6 @@ def bufr_to_ioda(config, logger): logger.debug(f"Checking subsets = {subsets}") # Get parameters from configuration - subsets = config["subsets"] data_format = config["data_format"] data_type = config["data_type"] data_description = config["data_description"] @@ -68,7 +67,7 @@ def bufr_to_ioda(config, logger): logger.info(f"reference_time = {reference_time}") - bufrfile = f"{cycle_type}.t{hh}z.{data_type}" + bufrfile = f"{cycle_type}.t{hh}z.{data_format}" DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) # ============================================ @@ -80,6 +79,7 @@ def bufr_to_ioda(config, logger): q = bufr.QuerySet(subsets) # MetaData + q.add('prepbufrDataLevelCategory', '*/PRSLEVEL/CAT') q.add('latitude', '*/PRSLEVEL/DRFTINFO/YDR') q.add('longitude', '*/PRSLEVEL/DRFTINFO/XDR') q.add('stationIdentification', '*/SID') @@ -88,16 +88,15 @@ def bufr_to_ioda(config, logger): q.add('releaseTime', '*/PRSLEVEL/DRFTINFO/HRDR') q.add('temperatureEventProgramCode', '*/PRSLEVEL/T___INFO/T__EVENT{1}/TPC') q.add('pressure', '*/PRSLEVEL/P___INFO/P__EVENT{1}/POB') + q.add('height', '*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZOB') # ObsValue - q.add('verticalSignificance', '*/PRSLEVEL/CAT') q.add('stationPressure', '*/PRSLEVEL/P___INFO/P__EVENT{1}/POB') q.add('airTemperature', '*/PRSLEVEL/T___INFO/T__EVENT{1}/TOB') q.add('virtualTemperature', '*/PRSLEVEL/T___INFO/TVO') q.add('specificHumidity', '*/PRSLEVEL/Q___INFO/Q__EVENT{1}/QOB') q.add('windEastward', '*/PRSLEVEL/W___INFO/W__EVENT{1}/UOB') q.add('windNorthward', '*/PRSLEVEL/W___INFO/W__EVENT{1}/VOB') - q.add('height', '*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZOB') # QualityMark q.add('pressureQM', '*/PRSLEVEL/P___INFO/P__EVENT{1}/PQM') @@ -123,17 +122,19 @@ def bufr_to_ioda(config, logger): logger.info('Executing QuerySet: get metadata') # MetaData - lat = r.get('latitude', 'verticalSignificance') - lon = r.get('longitude', 'verticalSignificance') + cat = r.get('prepbufrDataLevelCategory', 'prepbufrDataLevelCategory') + lat = r.get('latitude', 'prepbufrDataLevelCategory') + lon = r.get('longitude', 'prepbufrDataLevelCategory') lon[lon > 180] -= 360 # Convert Longitude from [0,360] to [-180,180] - sid = r.get('stationIdentification', 'verticalSignificance') - elv = r.get('stationElevation', 'verticalSignificance', type='float') - tpc = r.get('temperatureEventProgramCode', 'verticalSignificance') - pob = r.get('pressure', 'verticalSignificance') + sid = r.get('stationIdentification', 'prepbufrDataLevelCategory') + elv = r.get('stationElevation', 'prepbufrDataLevelCategory', type='float') + tpc = r.get('temperatureEventProgramCode', 'prepbufrDataLevelCategory') + pob = r.get('pressure', 'prepbufrDataLevelCategory') pob *= 100 + zob = r.get('height', 'prepbufrDataLevelCategory', type='float') # Time variable - hrdr = r.get('timeOffset', 'verticalSignificance') + hrdr = r.get('timeOffset', 'prepbufrDataLevelCategory', type='int64') ulan = r.get('releaseTime') ulan = np.int64(ulan*3600) @@ -143,33 +144,31 @@ def bufr_to_ioda(config, logger): ulan = ulan.reshape(ulan.shape) # ObsValue - cat = r.get('verticalSignificance', 'verticalSignificance') - ps = np.full(pob.shape[0], pob.fill_value) # Extract stationPressure from pressure, which belongs to CAT=1 + ps = np.full(pob.shape[0], pob.fill_value) # Extract stationPressure from pressure, which belongs to CAT=1 ps = np.where(cat == 0, pob, ps) - tob = r.get('airTemperature', 'verticalSignificance') + tob = r.get('airTemperature', 'prepbufrDataLevelCategory') tob += 273.15 - tsen = np.full(tob.shape[0], tob.fill_value) # Extract sensible temperature from tob, which belongs to TPC=1 - tsen = np.where(tpc == 1, tob, tsen) + tsen = np.full(tob.shape[0], tob.fill_value) # Extract sensible temperature from tob, which belongs to TPC=1 + tsen = np.where(((tpc >= 1) & (tpc < 8)), tob, tsen) tvo = np.full(tob.shape[0], tob.fill_value) # Extract virtual temperature from tob, which belongs to TPC <= 8 and TPC>1 - tvo = np.where(((tpc <= 8) & (tpc > 1)), tob, tvo) - qob = r.get('specificHumidity', 'verticalSignificance', type='float') + tvo = np.where((tpc == 8), tob, tvo) + qob = r.get('specificHumidity', 'prepbufrDataLevelCategory', type='float') qob *= 1.0e-6 - uob = r.get('windEastward', 'verticalSignificance') - vob = r.get('windNorthward', 'verticalSignificance') - zob = r.get('height', 'verticalSignificance') + uob = r.get('windEastward', 'prepbufrDataLevelCategory') + vob = r.get('windNorthward', 'prepbufrDataLevelCategory') # QualityMark - pobqm = r.get('pressureQM', 'verticalSignificance') + pobqm = r.get('pressureQM', 'prepbufrDataLevelCategory') psqm = np.full(pobqm.shape[0], pobqm.fill_value) # Extract stationPressureQM from pressureQM psqm = np.where(cat == 0, pobqm, psqm) - tobqm = r.get('airTemperatureQM', 'verticalSignificance') + tobqm = r.get('airTemperatureQM', 'prepbufrDataLevelCategory') tsenqm = np.full(tobqm.shape[0], tobqm.fill_value) # Extract airTemperature from tobqm, which belongs to TPC=1 - tsenqm = np.where(tpc == 1, tobqm, tsenqm) + tsenqm = np.where(((tpc >= 1) & (tpc < 8)), tobqm, tsenqm) tvoqm = np.full(tobqm.shape[0], tobqm.fill_value) # Extract virtual temperature from tob, which belongs to TPC <= 8 and TPC>1 - tvoqm = np.where(((tpc <= 8) & (tpc > 1)), tobqm, tvoqm) - qobqm = r.get('specificHumidityQM', 'verticalSignificance') - uobqm = r.get('windEastwardQM', 'verticalSignificance') - vobqm = r.get('windNorthwardQM', 'verticalSignificance') + tvoqm = np.where((tpc == 8), tobqm, tvoqm) + qobqm = r.get('specificHumidityQM', 'prepbufrDataLevelCategory') + uobqm = r.get('windEastwardQM', 'prepbufrDataLevelCategory') + vobqm = r.get('windNorthwardQM', 'prepbufrDataLevelCategory') logger.info('Executing QuerySet Done!') @@ -184,6 +183,7 @@ def bufr_to_ioda(config, logger): logger.debug(f' pob shape = {pob.shape}') logger.debug(f' hrdr shape = {hrdr.shape}') logger.debug(f' ulan shape = {ulan.shape}') + logger.debug(f' zob shape = {zob.shape}') logger.debug(f' ps shape = {ps.shape}') logger.debug(f' tob shape = {tob.shape}') @@ -192,7 +192,6 @@ def bufr_to_ioda(config, logger): logger.debug(f' qob shape = {qob.shape}') logger.debug(f' uob shape = {uob.shape}') logger.debug(f' vob shape = {vob.shape}') - logger.debug(f' zob shape = {zob.shape}') logger.debug(f' pobqm shape = {pobqm.shape}') logger.debug(f' psqm shape = {psqm.shape}') @@ -212,6 +211,7 @@ def bufr_to_ioda(config, logger): logger.debug(f' pob type = {pob.dtype}') logger.debug(f' hrdr type = {hrdr.dtype}') logger.debug(f' ulan type = {ulan.dtype}') + logger.debug(f' zob type = {zob.dtype}') logger.debug(f' ps type = {ps.dtype}') logger.debug(f' tob type = {tob.dtype}') @@ -220,7 +220,6 @@ def bufr_to_ioda(config, logger): logger.debug(f' qob type = {qob.dtype}') logger.debug(f' uob type = {uob.dtype}') logger.debug(f' vob type = {vob.dtype}') - logger.debug(f' zob type = {zob.dtype}') logger.debug(f' pobqm type = {pobqm.dtype}') logger.debug(f' psqm type = {psqm.dtype}') @@ -243,22 +242,20 @@ def bufr_to_ioda(config, logger): logger.info('Creating derived variables - dateTime from hrdr') cycleTimeSinceEpoch = np.int64(calendar.timegm(time.strptime(reference_time, '%Y-%m-%dT%H:%M:%SZ'))) - hrdr = Compute_dateTime(cycleTimeSinceEpoch, hrdr) + dateTime = Compute_dateTime(cycleTimeSinceEpoch, hrdr) logger.debug(' Check derived variables type ... ') - logger.debug(f' hrdr type = {hrdr.dtype}') + logger.debug(f' dateTime type = {dateTime.dtype}') end_time = time.time() running_time = end_time - start_time logger.info(f"Running time for creating derived variables : {running_time} seconds") # Create the dimensions - dims = { - 'Location': np.arange(0, lat.shape[0]) - } + dims = {'Location': np.arange(0, lat.shape[0])} # Create IODA ObsSpace - iodafile = f"{cycle_type}.t{hh}z.{data_type}.nc" + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{data_format}.nc" OUTPUT_PATH = os.path.join(ioda_dir, iodafile) logger.info(f"Create output file: {OUTPUT_PATH}") obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) @@ -270,6 +267,12 @@ def bufr_to_ioda(config, logger): # Create IODA variables logger.debug(' ... ... Create variables: name, type, units, and attributes') + # Prepbufr Data Level Category + obsspace.create_var('MetaData/prepbufrDataLevelCategory', dtype=cat.dtype, fillval=cat.fill_value) \ + .write_attr('units', '1') \ + .write_attr('long_name', 'Prepbufr Data Level Category') \ + .write_data(cat) + # Latitude obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat.fill_value) \ .write_attr('units', 'degrees_north') \ @@ -306,23 +309,24 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Pressure') \ .write_data(pob) + # Height of Observation + obsspace.create_var('MetaData/height', dtype=zob.dtype, fillval=zob.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height of Observation') \ + .write_data(zob) + # Datetime - obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=hrdr.fill_value) \ + obsspace.create_var('MetaData/dateTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ .write_attr('long_name', 'Datetime') \ - .write_data(hrdr) + .write_data(dateTime) # releaseTime - obsspace.create_var('MetaData/releaseTime', dtype=np.int64, fillval=hrdr.fill_value) \ + obsspace.create_var('MetaData/releaseTime', dtype=dateTime.dtype, fillval=dateTime.fill_value) \ .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ .write_attr('long_name', 'Release Time') \ .write_data(ulan) - # Prepbufr Data Level Category - obsspace.create_var('ObsValue/verticalSignificance', dtype=cat.dtype, fillval=cat.fill_value) \ - .write_attr('long_name', 'Prepbufr Data Level Category') \ - .write_data(cat) - # Station Pressure obsspace.create_var('ObsValue/stationPressure', dtype=pob.dtype, fillval=pob.fill_value) \ .write_attr('units', 'Pa') \ @@ -359,12 +363,6 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Northward Wind') \ .write_data(vob) - # Height of Observation - obsspace.create_var('ObsValue/height', dtype=zob.dtype, fillval=zob.fill_value) \ - .write_attr('units', 'm') \ - .write_attr('long_name', 'Height of Observation') \ - .write_data(zob) - # Pressure Quality Marker obsspace.create_var('QualityMarker/pressure', dtype=pobqm.dtype, fillval=pobqm.fill_value) \ .write_attr('long_name', 'Pressure Quality Marker') \ From eabf0a3a572ea4937f0006e427e6f52af6d016b8 Mon Sep 17 00:00:00 2001 From: Shastri Paturi Date: Thu, 7 Dec 2023 16:55:43 +0000 Subject: [PATCH 16/22] Update YAMLs for time window section (#798) PR for updating YAMLs for the time window section. close #759 --- parm/aero/hofx/hofx_nomodel.yaml | 5 +++-- parm/aero/variational/3dvar_fgat_gfs_aero.yaml | 5 +++-- parm/aero/variational/3dvar_gfs_aero.yaml | 5 +++-- parm/atm/hofx/hofx4d.yaml | 5 +++-- parm/atm/hofx/hofx_nomodel.yaml | 5 +++-- parm/atm/hofx/hofx_ufotest.yaml | 5 +++-- parm/atm/lgetkf/lgetkf.yaml | 5 +++-- parm/atm/variational/3dvar_dripcg.yaml | 5 +++-- parm/land/hofx/hofx_nomodel.yaml | 5 +++-- parm/land/letkfoi/letkfoi.yaml | 5 +++-- parm/soca/variational/3dvarfgat.yaml | 5 +++-- 11 files changed, 33 insertions(+), 22 deletions(-) diff --git a/parm/aero/hofx/hofx_nomodel.yaml b/parm/aero/hofx/hofx_nomodel.yaml index 81516f0e4..042c8f534 100644 --- a/parm/aero/hofx/hofx_nomodel.yaml +++ b/parm/aero/hofx/hofx_nomodel.yaml @@ -1,5 +1,6 @@ -window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' -window length: $(AERO_WINDOW_LENGTH) +time window: + begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' + length: $(AERO_WINDOW_LENGTH) geometry: fms initialization: namelist filename: $(DATA)/fv3jedi/fmsmpp.nml diff --git a/parm/aero/variational/3dvar_fgat_gfs_aero.yaml b/parm/aero/variational/3dvar_fgat_gfs_aero.yaml index 528f7e64f..5887ca55a 100644 --- a/parm/aero/variational/3dvar_fgat_gfs_aero.yaml +++ b/parm/aero/variational/3dvar_fgat_gfs_aero.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-FGAT - window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' - window length: $(AERO_WINDOW_LENGTH) + time window: + begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' + length: $(AERO_WINDOW_LENGTH) analysis variables: &3dvars [mass_fraction_of_sulfate_in_air, mass_fraction_of_hydrophobic_black_carbon_in_air, diff --git a/parm/aero/variational/3dvar_gfs_aero.yaml b/parm/aero/variational/3dvar_gfs_aero.yaml index 218f4c88d..719afad52 100644 --- a/parm/aero/variational/3dvar_gfs_aero.yaml +++ b/parm/aero/variational/3dvar_gfs_aero.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-Var - window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' - window length: $(AERO_WINDOW_LENGTH) + time window: + begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}' + length: $(AERO_WINDOW_LENGTH) analysis variables: &3dvars [mass_fraction_of_sulfate_in_air, mass_fraction_of_hydrophobic_black_carbon_in_air, diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml index 3c6e79004..bd961d55a 100644 --- a/parm/atm/hofx/hofx4d.yaml +++ b/parm/atm/hofx/hofx4d.yaml @@ -1,5 +1,6 @@ -window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' -window length: $(ATM_WINDOW_LENGTH) +time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) forecast length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index 3f442d517..ec3551100 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -1,5 +1,6 @@ -window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' -window length: $(ATM_WINDOW_LENGTH) +time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: namelist filename: $(DATA)/fv3jedi/fmsmpp.nml diff --git a/parm/atm/hofx/hofx_ufotest.yaml b/parm/atm/hofx/hofx_ufotest.yaml index 35b3250fa..a1f29a1ef 100644 --- a/parm/atm/hofx/hofx_ufotest.yaml +++ b/parm/atm/hofx/hofx_ufotest.yaml @@ -1,3 +1,4 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' -window end: '{{ATM_WINDOW_END}}' +time window: + begin: '{{ATM_WINDOW_BEGIN}}' + end: '{{ATM_WINDOW_END}}' observations: !INC ${OBS_LIST} diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index a84eefcee..d5086e3b2 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -10,8 +10,9 @@ geometry: npy: $(npy_ges) npz: $(npz_ges) field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml -window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' -window length: $(ATM_WINDOW_LENGTH) +time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] background: diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index abd945842..64f8ec8c0 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-Var - window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' - window length: $(ATM_WINDOW_LENGTH) + time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: fms initialization: diff --git a/parm/land/hofx/hofx_nomodel.yaml b/parm/land/hofx/hofx_nomodel.yaml index c0a9bc8d2..3d77fc300 100644 --- a/parm/land/hofx/hofx_nomodel.yaml +++ b/parm/land/hofx/hofx_nomodel.yaml @@ -1,5 +1,6 @@ -window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' -window length: $(LAND_WINDOW_LENGTH) +time window: + begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' + length: $(LAND_WINDOW_LENGTH) geometry: fms initialization: namelist filename: $(DATA)/fv3jedi/fmsmpp.nml diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index b1310504c..55fc33ebf 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -19,8 +19,9 @@ geometry: datapath: $(FIXgfs)/orog/${CASE}/ filename_orog: $(CASE)_oro_data.nc -window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' -window length: $(LAND_WINDOW_LENGTH) +time window: + begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' + length: $(LAND_WINDOW_LENGTH) background: datetime: '{{ current_cycle | to_isotime }}' diff --git a/parm/soca/variational/3dvarfgat.yaml b/parm/soca/variational/3dvarfgat.yaml index 1a893b091..e1590843e 100644 --- a/parm/soca/variational/3dvarfgat.yaml +++ b/parm/soca/variational/3dvarfgat.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-FGAT - window begin: '{{ATM_WINDOW_BEGIN}}' - window length: $(ATM_WINDOW_LENGTH) + time window: + begin: '{{ATM_WINDOW_BEGIN}}' + length: $(ATM_WINDOW_LENGTH) analysis variables: &soca_ana_vars [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh] geometry: From 0baa0b73c301544ec48062def8bd84d00dc4f04e Mon Sep 17 00:00:00 2001 From: Yaping Wang <49168260+ypwang19@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:48:54 -0600 Subject: [PATCH 17/22] Add the IODA converter for VIIRS AOD obs (#799) ### Description: 1. Added a converter for reading/writing aerosol optical depth (AOD) data from VIIRS to IODA. 2. Added/Edited a NetCDFToIodaConverter2d class to handle observations with different channels. 3. Added a ctest. Added VIIRS AOD sample data in testdata and sample YAML in testinput. --------- Co-authored-by: ypwang19 --- utils/obsproc/NetCDFToIodaConverter2D.h | 318 ++++++++++++++++++ utils/obsproc/Viirsaod2Ioda.h | 184 ++++++++++ .../applications/gdas_obsprovider2ioda.h | 4 + utils/test/CMakeLists.txt | 9 + utils/test/prepdata.sh | 2 + utils/test/testdata/viirs_aod_1.cdl | 180 ++++++++++ utils/test/testdata/viirs_aod_2.cdl | 192 +++++++++++ utils/test/testinput/gdas_viirsaod2ioda.yaml | 11 + 8 files changed, 900 insertions(+) create mode 100644 utils/obsproc/NetCDFToIodaConverter2D.h create mode 100644 utils/obsproc/Viirsaod2Ioda.h create mode 100644 utils/test/testdata/viirs_aod_1.cdl create mode 100644 utils/test/testdata/viirs_aod_2.cdl create mode 100644 utils/test/testinput/gdas_viirsaod2ioda.yaml diff --git a/utils/obsproc/NetCDFToIodaConverter2D.h b/utils/obsproc/NetCDFToIodaConverter2D.h new file mode 100644 index 000000000..0355fc4b9 --- /dev/null +++ b/utils/obsproc/NetCDFToIodaConverter2D.h @@ -0,0 +1,318 @@ +#pragma once + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "ioda/Engines/HH.h" +#include "ioda/Group.h" +#include "ioda/ObsGroup.h" + +#include "oops/mpi/mpi.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" +#include "oops/util/missingValues.h" + +namespace gdasapp { + // A simple data structure to organize the info to provide to the ioda + // writter + struct IodaVars2d { + int location_; // Number of observation per variable + int nVars_; // number of obs variables, should be set to + // for now in the children classes + int channel_; // number of channels + int nfMetadata_; // number of float metadata fields + int niMetadata_; // number of int metadata fields + + // Non optional metadata + Eigen::ArrayXf longitude_; // geo-location_ + Eigen::ArrayXf latitude_; // " + Eigen::Array datetime_; // Epoch date in seconds + std::string referenceDate_; // Reference date for epoch time + Eigen::Array channelNumber_; + + // Obs info + Eigen::ArrayXf obsVal_; // Observation value + Eigen::ArrayXf obsError_; // " error + Eigen::ArrayXi preQc_; // Quality control flag + + // Optional metadata + Eigen::ArrayXXf floatMetadata_; // Optional array of float metadata + std::vector floatMetadataName_; // String descriptor of the float metadata + Eigen::ArrayXXf intMetadata_; // Optional array of integer metadata + std::vector intMetadataName_; // String descriptor of the integer metadata + + // Optional global attributes + std::map strGlobalAttr_; + + // Constructor + explicit IodaVars2d(const int nobs = 0, + const int nchan = 1, + const std::vector fmnames = {}, + const std::vector imnames = {}) : + location_(nobs), nVars_(1), channel_(nchan), + nfMetadata_(fmnames.size()), niMetadata_(imnames.size()), + longitude_(location_), latitude_(location_), datetime_(location_), channelNumber_(channel_), + obsVal_(location_ * channel_), + obsError_(location_ * channel_), + preQc_(location_ * channel_), + floatMetadata_(location_, fmnames.size()), + floatMetadataName_(fmnames), + intMetadata_(location_, imnames.size()), + intMetadataName_(imnames) + { + oops::Log::trace() << "IodaVars2d::IodaVars2d created." << std::endl; + } + + // Append an other instance of IodaVars + void append(const IodaVars2d& other) { + // Check if the two instances can be concatenated + ASSERT(nVars_ == other.nVars_); + ASSERT(nfMetadata_ == other.nfMetadata_); + ASSERT(niMetadata_ == other.niMetadata_); + ASSERT(floatMetadataName_ == floatMetadataName_); + ASSERT(intMetadataName_ == intMetadataName_); + + // Concatenate Eigen arrays and vectors + longitude_.conservativeResize(location_ + other.location_); + latitude_.conservativeResize(location_ + other.location_); + datetime_.conservativeResize(location_ + other.location_); + obsVal_.conservativeResize(location_ * channel_ + other.location_ * other.channel_); + obsError_.conservativeResize(location_ * channel_ + other.location_ * other.channel_); + preQc_.conservativeResize(location_ * channel_ + other.location_ * other.channel_); + floatMetadata_.conservativeResize(location_ + other.location_, nfMetadata_); + intMetadata_.conservativeResize(location_ + other.location_, niMetadata_); + + // Copy data from the 'other' object to this object + longitude_.tail(other.location_) = other.longitude_; + latitude_.tail(other.location_) = other.latitude_; + datetime_.tail(other.location_) = other.datetime_; + obsVal_.tail(other.location_ * other.channel_) = other.obsVal_; + obsError_.tail(other.location_ * other.channel_) = other.obsError_; + preQc_.tail(other.location_ * other.channel_) = other.preQc_; + floatMetadata_.bottomRows(other.location_) = other.floatMetadata_; + intMetadata_.bottomRows(other.location_) = other.intMetadata_; + + // Update obs count + location_ += other.location_; + oops::Log::trace() << "IodaVars2d::IodaVars2d done appending." << std::endl; + } + }; + + // Base class for the converters + class NetCDFToIodaConverter2d { + public: + // Constructor: Stores the configuration as a data members + explicit NetCDFToIodaConverter2d(const eckit::Configuration & fullConfig, + const eckit::mpi::Comm & comm): comm_(comm), + fullConfig_(fullConfig) { + // time window info + std::string winbegin; + std::string winend; + fullConfig.get("window begin", winbegin); + fullConfig.get("window end", winend); + windowBegin_ = util::DateTime(winbegin); + windowEnd_ = util::DateTime(winend); + variable_ = "None"; + oops::Log::info() << "--- Window begin: " << winbegin << std::endl; + oops::Log::info() << "--- Window end: " << winend << std::endl; + + // get input netcdf files + fullConfig.get("input files", inputFilenames_); + oops::Log::info() << "--- Input files: " << inputFilenames_ << std::endl; + + // ioda output file name + outputFilename_ = fullConfig.getString("output file"); + oops::Log::info() << "--- Output files: " << outputFilename_ << std::endl; + oops::Log::trace() << "NetCDFToIodaConverter::NetCDFToIodaConverter created." << std::endl; + } + + // Method to write out a IODA file (writter called in destructor) + void writeToIoda() { + // Extract ioda variables from the provider's files + int myrank = comm_.rank(); + int nobs(0); + + // Currently need the PE count to be less than the number of files + ASSERT(comm_.size() <= inputFilenames_.size()); + + // Read the provider's netcdf file + gdasapp::IodaVars2d iodaVars = providerToIodaVars(inputFilenames_[myrank]); + for (int i = myrank + comm_.size(); i < inputFilenames_.size(); i += comm_.size()) { + iodaVars.append(providerToIodaVars(inputFilenames_[i])); + oops::Log::info() << " appending: " << inputFilenames_[i] << std::endl; + oops::Log::info() << " obs count: " << iodaVars.location_ << std::endl; + } + nobs = iodaVars.location_; + + // Get the total number of obs across pe's + int nobsAll(0); + comm_.allReduce(nobs, nobsAll, eckit::mpi::sum()); + gdasapp::IodaVars2d iodaVarsAll(nobsAll, + iodaVars.channel_, + iodaVars.floatMetadataName_, + iodaVars.intMetadataName_); + + // Gather iodaVars arrays + gatherObs(iodaVars.longitude_, iodaVarsAll.longitude_); + gatherObs(iodaVars.latitude_, iodaVarsAll.latitude_); + gatherObs(iodaVars.datetime_, iodaVarsAll.datetime_); + gatherObs(iodaVars.obsVal_, iodaVarsAll.obsVal_); + gatherObs(iodaVars.obsError_, iodaVarsAll.obsError_); + gatherObs(iodaVars.preQc_, iodaVarsAll.preQc_); + + // Create empty group backed by HDF file + if (oops::mpi::world().rank() == 0) { + ioda::Group group = + ioda::Engines::HH::createFile(outputFilename_, + ioda::Engines::BackendCreateModes::Truncate_If_Exists); + + // Update the group with the location dimension + ioda::NewDimensionScales_t + newDims { + ioda::NewDimensionScale("Location", iodaVarsAll.location_), + ioda::NewDimensionScale("Channel", iodaVarsAll.channel_) + }; + + ioda::ObsGroup ogrp = ioda::ObsGroup::generate(group, newDims); + + ogrp.vars["Channel"].writeWithEigenRegular(iodaVars.channelNumber_); + + // Set up the creation parameters + ioda::VariableCreationParameters float_params = createVariableParams(); + ioda::VariableCreationParameters int_params = createVariableParams(); + ioda::VariableCreationParameters long_params = createVariableParams(); + + // Create the mendatory IODA variables + ioda::Variable iodaDatetime = + ogrp.vars.createWithScales("MetaData/dateTime", + {ogrp.vars["Location"]}, long_params); + iodaDatetime.atts.add("units", {iodaVars.referenceDate_}, {1}); + ioda::Variable iodaLat = + ogrp.vars.createWithScales("MetaData/latitude", + {ogrp.vars["Location"]}, float_params); + ioda::Variable iodaLon = + ogrp.vars.createWithScales("MetaData/longitude", + {ogrp.vars["Location"]}, float_params); + + ioda::Variable iodaObsVal = + ogrp.vars.createWithScales("ObsValue/"+variable_, + {ogrp.vars["Location"], + ogrp.vars["Channel"]}, float_params); + ioda::Variable iodaObsErr = + ogrp.vars.createWithScales("ObsError/"+variable_, + {ogrp.vars["Location"], + ogrp.vars["Channel"]}, float_params); + ioda::Variable iodaPreQc = + ogrp.vars.createWithScales("PreQC/"+variable_, + {ogrp.vars["Location"], + ogrp.vars["Channel"]}, int_params); + + // add input filenames to IODA file global attributes + ogrp.atts.add("obs_source_files", inputFilenames_); + + // add global attributes collected from the specific converter + for (const auto& globalAttr : iodaVars.strGlobalAttr_) { + ogrp.atts.add(globalAttr.first , globalAttr.second); + } + + // Create the optional IODA integer metadata + ioda::Variable tmpIntMeta; + int count = 0; + for (const std::string& strMeta : iodaVars.intMetadataName_) { + tmpIntMeta = ogrp.vars.createWithScales("MetaData/"+strMeta, + {ogrp.vars["Location"]}, int_params); + tmpIntMeta.writeWithEigenRegular(iodaVars.intMetadata_.col(count)); + count++; + } + + // Create the optional IODA float metadata + ioda::Variable tmpFloatMeta; + count = 0; + for (const std::string& strMeta : iodaVars.floatMetadataName_) { + tmpFloatMeta = ogrp.vars.createWithScales("MetaData/"+strMeta, + {ogrp.vars["Location"]}, float_params); + tmpFloatMeta.writeWithEigenRegular(iodaVars.floatMetadata_.col(count)); + count++; + } + + // Write obs info to group + oops::Log::info() << "Writing ioda file" << std::endl; + iodaLon.writeWithEigenRegular(iodaVarsAll.longitude_); + iodaLat.writeWithEigenRegular(iodaVarsAll.latitude_); + iodaDatetime.writeWithEigenRegular(iodaVarsAll.datetime_); + iodaObsVal.writeWithEigenRegular(iodaVarsAll.obsVal_); + iodaObsErr.writeWithEigenRegular(iodaVarsAll.obsError_); + iodaPreQc.writeWithEigenRegular(iodaVarsAll.preQc_); + } + } + + private: + // Virtual method that reads the provider's netcdf file and store the relevant + // info in a IodaVars struct + virtual gdasapp::IodaVars2d providerToIodaVars(const std::string fileName) = 0; + + // Gather for eigen array + template + void gatherObs(const Eigen::Array & obsPe, + Eigen::Array & obsAllPes) { + // define root pe + const size_t root = 0; + + // send pointer to the PE's data + std::vector send(obsPe.data(), obsPe.data() + obsPe.size()); + size_t ntasks = comm_.size(); + + // gather the sizes of the send buffers + int localnobs = send.size(); + std::vector sizes(ntasks); + comm_.allGather(localnobs, sizes.begin(), sizes.end()); + + // displacement for the received data + std::vector displs(ntasks); + size_t rcvsz = sizes[0]; + displs[0] = 0; + for (size_t jj = 1; jj < ntasks; ++jj) { + displs[jj] = displs[jj - 1] + sizes[jj - 1]; + rcvsz += sizes[jj]; + } + + // create receiving buffer + std::vector recv(0); + + // gather all send buffers + if (comm_.rank() == root) recv.resize(rcvsz); + comm_.barrier(); + comm_.gatherv(send, recv, sizes, displs, root); + + if (comm_.rank() == root) { + obsAllPes.segment(0, recv.size()) = + Eigen::Map>(recv.data(), recv.size()); + } + } + + // Short-cut to create type dependent VariableCreationParameters + template + ioda::VariableCreationParameters createVariableParams() { + ioda::VariableCreationParameters params; + params.chunk = true; // allow chunking + params.compressWithGZIP(); // compress using gzip + params.setFillValue(util::missingValue()); + + return params; + } + + protected: + util::DateTime windowBegin_; + util::DateTime windowEnd_; + std::vector inputFilenames_; + std::string outputFilename_; + std::string variable_; + const eckit::mpi::Comm & comm_; + const eckit::Configuration & fullConfig_; + }; +} // namespace gdasapp diff --git a/utils/obsproc/Viirsaod2Ioda.h b/utils/obsproc/Viirsaod2Ioda.h new file mode 100644 index 000000000..4bea0bd85 --- /dev/null +++ b/utils/obsproc/Viirsaod2Ioda.h @@ -0,0 +1,184 @@ +#pragma once + +#include +#include // NOLINT (using C API) +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include // NOLINT + +#include "ioda/Group.h" +#include "ioda/ObsGroup.h" + +#include "NetCDFToIodaConverter2D.h" +#include "superob.h" + +namespace gdasapp { + + class Viirsaod2Ioda : public NetCDFToIodaConverter2d { + public: + explicit Viirsaod2Ioda(const eckit::Configuration & fullConfig, const eckit::mpi::Comm & comm) + : NetCDFToIodaConverter2d(fullConfig, comm) { + variable_ = "aerosolOpticalDepth"; + } + + // Read netcdf file and populate iodaVars + gdasapp::IodaVars2d providerToIodaVars(const std::string fileName) final { + oops::Log::info() << "Processing files provided by VIIRSAOD" << std::endl; + + // Open the NetCDF file in read-only mode + netCDF::NcFile ncFile(fileName, netCDF::NcFile::read); + oops::Log::info() << "Reading... " << fileName << std::endl; + + // Get dimensions + int dimRow = ncFile.getDim("Rows").getSize(); + int dimCol = ncFile.getDim("Columns").getSize(); + oops::Log::info() << "row,col " << dimRow << dimCol << std::endl; + + // Read lat and lon + float lon2d[dimRow][dimCol]; + ncFile.getVar("Longitude").getVar(lon2d); + + float lat2d[dimRow][dimCol]; + ncFile.getVar("Latitude").getVar(lat2d); + + float aod550[dimRow][dimCol]; + ncFile.getVar("AOD550").getVar(aod550); + const float missingValue = -999.999; + + int8_t qcall[dimRow][dimCol]; + ncFile.getVar("QCAll").getVar(qcall); + + int8_t qcpath[dimRow][dimCol]; + ncFile.getVar("QCPath").getVar(qcpath); + + // string obstime + std::string time_coverage_end; + ncFile.getAtt("time_coverage_end").getValues(time_coverage_end); + oops::Log::info() << "time_coverage_end type " << time_coverage_end << std::endl; + std::tm timeinfo = {}; + // Create an input string stream and parse the string + std::istringstream dateStream(time_coverage_end); + dateStream >> std::get_time(&timeinfo, "%Y-%m-%dT%H:%M:%SZ"); + + std::tm referenceTime = {}; + referenceTime.tm_year = 70; // 1970 + referenceTime.tm_mon = 0; // January (months are 0-based) + referenceTime.tm_mday = 1; // 1st day of the month + referenceTime.tm_hour = 0; + referenceTime.tm_min = 0; + referenceTime.tm_sec = 0; + + std::time_t timestamp = std::mktime(&timeinfo); + std::time_t referenceTimestamp = std::mktime(&referenceTime); + std::time_t secondsSinceReference = std::difftime(timestamp, referenceTimestamp); + + + // Apply scaling/unit change and compute the necessary fields + std::vector> mask(dimRow, std::vector(dimCol)); + std::vector> obsvalue(dimRow, std::vector(dimCol)); + std::vector> obserror(dimRow, std::vector(dimCol)); + std::vector> preqc(dimRow, std::vector(dimCol)); + std::vector> seconds(dimRow, std::vector(dimCol)); + std::vector> lat(dimRow, std::vector(dimCol)); + std::vector> lon(dimRow, std::vector(dimCol)); + + + // superobing, not work yet... + if (false) { + std::vector> lon2d_s = + gdasapp::superobutils::subsample2D(lon, mask, fullConfig_); + std::vector> lat2d_s = + gdasapp::superobutils::subsample2D(lat, mask, fullConfig_); + std::vector> obsvalue_s = + gdasapp::superobutils::subsample2D(obsvalue, mask, fullConfig_); + std::vector> obserror_s = + gdasapp::superobutils::subsample2D(obserror, mask, fullConfig_); + std::vector> seconds_s = + gdasapp::superobutils::subsample2D(seconds, mask, fullConfig_); + } + + // Thinning + float thinThreshold; + fullConfig_.get("thinning.threshold", thinThreshold); + oops::Log::info() << " thinthreshold " << thinThreshold << std::endl; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis(0.0, 1.0); + + // Create thinning and missing value mask + int nobs(0); + for (int i = 0; i < dimRow; i++) { + for (int j = 0; j < dimCol; j++) { + if (aod550[i][j] != missingValue) { + // Random number generation for thinning + float isThin = dis(gen); + if (isThin > thinThreshold) { + preqc[i][j] = static_cast(qcall[i][j]); + obsvalue[i][j] = static_cast(aod550[i][j]); + lat[i][j] = lat2d[i][j]; + lon[i][j] = lon2d[i][j]; + // dark land + float obserrorValue = 0.111431 + 0.128699 * static_cast(aod550[i][j]); + // ocean + if (qcpath[i][j] % 2 == 1) { + obserrorValue = 0.00784394 + 0.219923 * static_cast(aod550[i][j]); + } + // bright land + if (qcpath[i][j] % 4 == 2) { + obserrorValue = 0.0550472 + 0.299558 * static_cast(aod550[i][j]); + } + obserror[i][j] = obserrorValue; + mask[i][j] = 1; + nobs += 1; + } + } + } + } + + + // read in channel number + std::string channels; + fullConfig_.get("channel", channels); + std::istringstream ss(channels); + std::vector channelNumber; + std::string substr; + while (std::getline(ss, substr, ',')) { + int intValue = std::stoi(substr); + channelNumber.push_back(intValue); + } + oops::Log::info() << " channels " << channelNumber << std::endl; + int nchan(channelNumber.size()); + oops::Log::info() << " number of channels " << nchan << std::endl; + // Create instance of iodaVars object + gdasapp::IodaVars2d iodaVars(nobs, nchan, {}, {}); + + oops::Log::info() << " eigen... " << std::endl; + // Store into eigen arrays + for (int k = 0; k < nchan; k++) { + iodaVars.channelNumber_(k) = channelNumber[k]; + int loc(0); + for (int i = 0; i < dimRow; i++) { + for (int j = 0; j < dimCol; j++) { + if (mask[i][j] == 1) { // mask apply to all channels + iodaVars.longitude_(loc) = lon[i][j]; + iodaVars.latitude_(loc) = lat[i][j]; + iodaVars.datetime_(loc) = secondsSinceReference; + iodaVars.referenceDate_ = "1970-01-01 00:00:00"; + // VIIRS AOD use only one channel (4) + iodaVars.obsVal_(nchan*loc+k) = obsvalue[i][j]; + iodaVars.preQc_(nchan*loc+k) = preqc[i][j]; + iodaVars.obsError_(nchan*loc+k) = obserror[i][j]; + loc += 1; + } + } + } + oops::Log::info() << " total location " << loc << std::endl; + } + + return iodaVars; + }; + }; // class Viirsaod2Ioda +} // namespace gdasapp diff --git a/utils/obsproc/applications/gdas_obsprovider2ioda.h b/utils/obsproc/applications/gdas_obsprovider2ioda.h index 3d4dc1a7c..2fef42183 100644 --- a/utils/obsproc/applications/gdas_obsprovider2ioda.h +++ b/utils/obsproc/applications/gdas_obsprovider2ioda.h @@ -11,6 +11,7 @@ #include "../Rads2Ioda.h" #include "../Smap2Ioda.h" #include "../Smos2Ioda.h" +#include "../Viirsaod2Ioda.h" namespace gdasapp { class ObsProvider2IodaApp : public oops::Application { @@ -39,6 +40,9 @@ namespace gdasapp { } else if (provider == "AMSR2") { IcecAmsr2Ioda conv2ioda(fullConfig, this->getComm()); conv2ioda.writeToIoda(); + } else if (provider == "VIIRSAOD") { + Viirsaod2Ioda conv2ioda(fullConfig, this->getComm()); + conv2ioda.writeToIoda(); } else { oops::Log::info() << "Provider not implemented" << std::endl; return 1; diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index a53c92c2e..cf74db8aa 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -6,6 +6,7 @@ list( APPEND utils_test_input testinput/gdas_smap2ioda.yaml testinput/gdas_smos2ioda.yaml testinput/gdas_icecamsr2ioda.yaml + testinput/gdas_viirsaod2ioda.yaml ) set( gdas_utils_test_ref @@ -14,6 +15,7 @@ set( gdas_utils_test_ref testref/smap2ioda.test testref/smos2ioda.test testref/icecamsr2ioda.test + testref/viirsaod2ioda.test ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput) @@ -76,6 +78,13 @@ ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda LIBS gdas-utils WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +# Test the VIIRS AOD to IODA converter +ecbuild_add_test( TARGET test_gdasapp_util_viirsaod2ioda + COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x + ARGS "../testinput/gdas_viirsaod2ioda.yaml" + LIBS gdas-utils + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + # Test the AMSR2 to IODA converter # TODO(Mindo): Turn back on when date is fixed #ecbuild_add_test( TARGET test_gdasapp_util_icecamsr2ioda diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh index f6a1c0698..b7ca4a96e 100755 --- a/utils/test/prepdata.sh +++ b/utils/test/prepdata.sh @@ -29,3 +29,5 @@ cdl2nc4 sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl cdl2nc4 sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl cdl2nc4 ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl cdl2nc4 ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl +cdl2nc4 viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl +cdl2nc4 viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl diff --git a/utils/test/testdata/viirs_aod_1.cdl b/utils/test/testdata/viirs_aod_1.cdl new file mode 100644 index 000000000..0b3cee0e1 --- /dev/null +++ b/utils/test/testdata/viirs_aod_1.cdl @@ -0,0 +1,180 @@ +netcdf viirs_aod { +dimensions: + Rows = 10 ; + Columns = 10 ; +variables: + float AOD550(Rows, Columns) ; + AOD550:long_name = "Aerosol optical depth at 550 nm" ; + AOD550:coordinates = "Longitude Latitude" ; + AOD550:units = "1" ; + AOD550:_FillValue = -999.999f ; + AOD550:valid_range = -0.05f, 5.f ; + float Latitude(Rows, Columns) ; + Latitude:long_name = "Latitude" ; + Latitude:units = "degrees_north" ; + Latitude:comments = "Pixel latitude in field Latitude (degree)" ; + Latitude:_FillValue = -999.f ; + Latitude:valid_range = -90.f, 90.f ; + float Longitude(Rows, Columns) ; + Longitude:long_name = "Longitude" ; + Longitude:units = "degrees_east" ; + Longitude:comments = "Pixel longitude in field Longitude (degree)" ; + Longitude:_FillValue = -999.f ; + Longitude:valid_range = -180.f, 180.f ; + byte QCAll(Rows, Columns) ; + QCAll:long_name = "Retrieval quality: 0: high; 1: medium; 2: low; 3: no retrieval" ; + QCAll:coordinates = "Longitude Latitude" ; + QCAll:units = "1" ; + QCAll:_FillValue = -128b ; + QCAll:valid_range = 0b, 3b ; + byte QCPath(Rows, Columns) ; + QCPath:long_name = "Flags for retrieval path (0-No/1-Yes): bit 0: retrieval over water; bit 1: over bright land; bit 2: over glint water; bit 3: retrieval with SW scheme over land; bit 4: retrieval with SWIR scheme over land; bit 5: retrieval over bright-land algorithm" ; + QCPath:coordinates = "Longitude Latitude" ; + QCPath:units = "1" ; + QCPath:_FillValue = -128b ; + QCPath:valid_range = 0b, 31b ; + float Residual(Rows, Columns) ; + Residual:long_name = "Retrieval residual of the best solution" ; + Residual:coordinates = "Longitude Latitude" ; + Residual:units = "1" ; + Residual:_FillValue = -999.999f ; + Residual:valid_range = 0.f, 999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Dataset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "S-NPP Data Exploitation" ; + :institution = "DOC/NOAA/NESDIS/NDE->S-NPP Data Exploitation, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :satellite_name = "NPP" ; + :instrument_name = "VIIRS" ; + :title = "JPSS Risk Reduction Unique Aerosol Optical Depth" ; + :summary = "Aerosol Optical Depth" ; + :history = "Tue May 19 10:48:26 2020: ncks -v Latitude,Longitude,AOD550,QCPath,Residual,QCAll -d Columns,337,346 -d Rows,268,277 sample_viirs_class.nc sample_subset_testcase.nc\nVIIRS AOD Version 1.0" ; + :processing_level = "NOAA Level 2" ; + :references = "" ; + :id = "ad1a951f-cc67-45d5-ab9d-b7be77d6a055" ; + :Metadata_Link = "JRR-AOD_v1r1_npp_s201804150418347_e201804150419589_c201804150512090.nc" ; + :start_orbit_number = 33494 ; + :end_orbit_number = 33494 ; + :day_night_data_flag = "day" ; + :ascend_descend_data_flag = 0 ; + :time_coverage_start = "2018-04-15T04:18:34Z" ; + :time_coverage_end = "2018-04-15T04:19:58Z" ; + :date_created = "2018-04-15T05:12:12Z" ; + :cdm_data_type = "swath" ; + :geospatial_first_scanline_first_fov_lat = 48.43998f ; + :geospatial_first_scanline_last_fov_lat = 42.56979f ; + :geospatial_last_scanline_first_fov_lat = 53.45447f ; + :geospatial_last_scanline_last_fov_lat = 47.02182f ; + :geospatial_first_scanline_first_fov_lon = 147.643f ; + :geospatial_first_scanline_last_fov_lon = 108.7127f ; + :geospatial_last_scanline_first_fov_lon = 147.7868f ; + :geospatial_last_scanline_last_fov_lon = 105.0699f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :geospatial_bounds = "POLYGON((147.643 48.44,108.713 42.5698,105.07 47.0218,147.787 53.4545,147.643 48.44))" ; + :NCO = "netCDF Operators version 4.9.1 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + AOD550 = + 0.1204824, 0.1181651, 0.1153356, 0.1151632, 0.113657, 0.1331273, 0.1048789, + 0.1148486, 0.133357, 0.1349388, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + 0.1216023, 0.1633492, 0.1189802, 0.1597583, 0.104865, 0.1156261, 0.1624881, + 0.1063675, 0.1329711, 0.1083063, + 0.1617547, 0.1069087, 0.1154101, 0.1578954, 0.1340958, 0.1618325, + 0.1060463, 0.134523, 0.1330938, 0.09775206, + 0.1156879, 0.1049347, 0.0976741, 0.1123927, 0.1133153, 0.1134829, 0.112356, + 0.1141562, 0.1297655, 0.105854 ; + + Latitude = + 50.28726, 50.28702, 50.28678, 50.28655, 50.28631, 50.28606, 50.28583, + 50.28558, 50.28534, 50.2851, + 50.299, 50.29874, 50.29851, 50.29826, 50.29802, 50.29776, 50.29752, + 50.29727, 50.29702, 50.29676, + 50.31073, 50.31047, 50.31023, 50.30998, 50.30972, 50.30947, 50.30921, + 50.30896, 50.3087, 50.30844, + 50.32247, 50.32221, 50.32196, 50.3217, 50.32143, 50.32117, 50.32091, + 50.32065, 50.32038, 50.32011, + 50.25031, 50.25015, 50.24999, 50.24984, 50.24968, 50.24952, 50.24934, + 50.24919, 50.24903, 50.24885, + 50.26204, 50.26187, 50.26171, 50.26155, 50.26138, 50.26121, 50.26104, + 50.26087, 50.26069, 50.26053, + 50.27377, 50.27361, 50.27343, 50.27325, 50.27308, 50.27291, 50.27274, + 50.27255, 50.27237, 50.27219, + 50.2855, 50.28532, 50.28515, 50.28497, 50.28478, 50.28461, 50.28442, + 50.28424, 50.28405, 50.28386, + 50.29723, 50.29705, 50.29686, 50.29668, 50.29649, 50.2963, 50.29611, + 50.29593, 50.29573, 50.29554, + 50.30896, 50.30877, 50.30858, 50.30839, 50.3082, 50.308, 50.3078, 50.30761, + 50.30741, 50.30721 ; + + Longitude = + 141.8715, 141.8586, 141.8457, 141.8328, 141.82, 141.8071, 141.7943, + 141.7815, 141.7687, 141.7559, + 141.8712, 141.8583, 141.8454, 141.8325, 141.8196, 141.8068, 141.794, + 141.7811, 141.7684, 141.7556, + 141.8709, 141.858, 141.8451, 141.8322, 141.8193, 141.8065, 141.7937, + 141.7808, 141.7681, 141.7553, + 141.8706, 141.8577, 141.8448, 141.8319, 141.819, 141.8062, 141.7933, + 141.7805, 141.7677, 141.7549, + 141.8644, 141.8515, 141.8386, 141.8257, 141.8129, 141.8, 141.7872, + 141.7744, 141.7617, 141.7489, + 141.864, 141.8511, 141.8382, 141.8253, 141.8125, 141.7997, 141.7868, + 141.774, 141.7612, 141.7485, + 141.8637, 141.8508, 141.8379, 141.825, 141.8121, 141.7993, 141.7865, + 141.7736, 141.7608, 141.7481, + 141.8633, 141.8504, 141.8375, 141.8246, 141.8118, 141.7989, 141.7861, + 141.7733, 141.7605, 141.7477, + 141.8629, 141.85, 141.8371, 141.8242, 141.8114, 141.7985, 141.7857, + 141.7729, 141.7601, 141.7473, + 141.8626, 141.8497, 141.8368, 141.8239, 141.811, 141.7981, 141.7853, + 141.7725, 141.7597, 141.7469 ; + + QCAll = + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; + + QCPath = + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ; + + Residual = + 0.01209862, 0.003792281, 0.008564534, 0.01330118, 0.01130516, 0.008588978, + 0.01217207, 0.01069431, 0.01022208, 0.02603927, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + 0.01604747, 0.007362799, 0.01586136, 0.01100692, 0.009566146, 0.01119804, + 0.01212741, 0.01139778, 0.008210129, 0.0106705, + 0.01781144, 0.01445681, 0.01132793, 0.01064146, 0.009726279, 0.007063937, + 0.01453139, 0.01188723, 0.01142784, 0.01012625, + 0.007868396, 0.01640957, 0.0178295, 0.01041934, 0.008885228, 0.01041434, + 0.01380985, 0.01131611, 0.01026587, 0.01259002 ; +} diff --git a/utils/test/testdata/viirs_aod_2.cdl b/utils/test/testdata/viirs_aod_2.cdl new file mode 100644 index 000000000..faad15ff1 --- /dev/null +++ b/utils/test/testdata/viirs_aod_2.cdl @@ -0,0 +1,192 @@ +netcdf viirs_aod { +dimensions: + Rows = 10 ; + Columns = 10 ; +variables: + float AOD550(Rows, Columns) ; + AOD550:long_name = "Aerosol optical depth at 550 nm" ; + AOD550:coordinates = "Longitude Latitude" ; + AOD550:units = "1" ; + AOD550:_FillValue = -999.999f ; + AOD550:valid_range = -0.05f, 5.f ; + float Latitude(Rows, Columns) ; + Latitude:long_name = "Latitude" ; + Latitude:units = "degrees_north" ; + Latitude:comments = "Pixel latitude in field Latitude (degree)" ; + Latitude:_FillValue = -999.f ; + Latitude:valid_range = -90.f, 90.f ; + float Longitude(Rows, Columns) ; + Longitude:long_name = "Longitude" ; + Longitude:units = "degrees_east" ; + Longitude:comments = "Pixel longitude in field Longitude (degree)" ; + Longitude:_FillValue = -999.f ; + Longitude:valid_range = -180.f, 180.f ; + byte QCAll(Rows, Columns) ; + QCAll:long_name = "Retrieval quality: 0: high; 1: medium; 2: low; 3: no retrieval" ; + QCAll:coordinates = "Longitude Latitude" ; + QCAll:units = "1" ; + QCAll:_FillValue = -128b ; + QCAll:valid_range = 0b, 3b ; + byte QCPath(Rows, Columns) ; + QCPath:long_name = "Flags for retrieval path (0-No/1-Yes): bit 0: retrieval over water; bit 1: over bright land; bit 2: over glint water; bit 3: retrieval with SW scheme over land; bit 4: retrieval with SWIR scheme over land; bit 5: retrieval over bright-land algorithm" ; + QCPath:coordinates = "Longitude Latitude" ; + QCPath:units = "1" ; + QCPath:_FillValue = -128b ; + QCPath:valid_range = 0b, 31b ; + float Residual(Rows, Columns) ; + Residual:long_name = "Retrieval residual of the best solution" ; + Residual:coordinates = "Longitude Latitude" ; + Residual:units = "1" ; + Residual:_FillValue = -999.999f ; + Residual:valid_range = 0.f, 999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Dataset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "S-NPP Data Exploitation" ; + :institution = "DOC/NOAA/NESDIS/NDE->S-NPP Data Exploitation, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :satellite_name = "NPP" ; + :instrument_name = "VIIRS" ; + :title = "JPSS Risk Reduction Unique Aerosol Optical Depth" ; + :summary = "Aerosol Optical Depth" ; + :history = "Tue May 19 10:48:26 2020: ncks -v Latitude,Longitude,AOD550,QCPath,Residual,QCAll -d Columns,337,346 -d Rows,268,277 sample_viirs_class.nc sample_subset_testcase.nc\nVIIRS AOD Version 1.0" ; + :processing_level = "NOAA Level 2" ; + :references = "" ; + :id = "ad1a951f-cc67-45d5-ab9d-b7be77d6a055" ; + :Metadata_Link = "JRR-AOD_v1r1_npp_s201804150418347_e201804150419589_c201804150512090.nc" ; + :start_orbit_number = 33494 ; + :end_orbit_number = 33494 ; + :day_night_data_flag = "day" ; + :ascend_descend_data_flag = 0 ; + :time_coverage_start = "2018-04-15T04:18:34Z" ; + :time_coverage_end = "2018-04-15T04:19:58Z" ; + :date_created = "2018-04-15T05:12:12Z" ; + :cdm_data_type = "swath" ; + :geospatial_first_scanline_first_fov_lat = 48.43998f ; + :geospatial_first_scanline_last_fov_lat = 42.56979f ; + :geospatial_last_scanline_first_fov_lat = 53.45447f ; + :geospatial_last_scanline_last_fov_lat = 47.02182f ; + :geospatial_first_scanline_first_fov_lon = 147.643f ; + :geospatial_first_scanline_last_fov_lon = 108.7127f ; + :geospatial_last_scanline_first_fov_lon = 147.7868f ; + :geospatial_last_scanline_last_fov_lon = 105.0699f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :geospatial_bounds = "POLYGON((147.643 48.44,108.713 42.5698,105.07 47.0218,147.787 53.4545,147.643 48.44))" ; + :NCO = "netCDF Operators version 4.9.1 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + AOD550 = + 0.08672047, 0.1280901, 0.01021459, 0.06967962, 0.1172924, 0.1077904, + 0.0879033, 0.005564162, 0.05259846, 0.0844786, + 0.04495948, 0.07685012, 0.10102, 0.02387906, 0.007571651, 0.05067238, + 0.01620957, 0.02683542, 0.1328741, 0.05480475, + 0.0328154, 0.1128744, 0.01603194, 0.1116905, 0.07046779, 0.08973835, + 0.02214303, 0.02760522, 0.09676082, 0.007294201, + 0.03729188, 0.08136128, 0.034016, 0.05721173, 0.1383349, 0.1388035, + 0.08501249, 0.08002064, 0.002229004, 0.1466849, + 0.08595434, 0.1187636, 0.0842336, 0.1316003, 0.08762938, 0.1063275, + 0.02228002, 0.06426761, 0.1040835, 0.01569296, + 0.06594078, 0.02493032, 0.07604679, 0.1228554, 0.01351601, 0.1200103, + 0.08476895, 0.08840216, 0.0297151, 0.06541774, + 0.04438556, 0.005633651, 0.004602726, 0.06796575, 0.1117296, 0.08359431, + 0.05776704, 0.02521092, 0.1257392, 0.08985777, + 0.1174072, 0.1272764, 0.09047445, 0.1171591, 0.09236053, 0.003174779, + 0.1125697, 0.02640632, 0.06877713, 0.07696841, + 0.07260314, 0.1266579, 0.02622208, 0.002195231, 0.1273146, 0.1114012, + 0.06850463, 0.06253476, 0.01750943, 0.05080187, + 0.01419886, 0.1073746, 0.01156281, 0.03089254, 0.08606643, 0.04407473, + 0.09835901, 0.1205353, 0.05268203, 0.01401606 ; + + Latitude = + 51.28726, 51.28702, 51.28678, 51.28655, 51.28631, 51.28606, 51.28583, + 51.28558, 51.28534, 51.2851, + 51.299, 51.29874, 51.29851, 51.29826, 51.29802, 51.29776, 51.29752, + 51.29727, 51.29702, 51.29676, + 51.31073, 51.31047, 51.31023, 51.30998, 51.30972, 51.30947, 51.30921, + 51.30896, 51.3087, 51.30844, + 51.32247, 51.32221, 51.32196, 51.3217, 51.32143, 51.32117, 51.32091, + 51.32065, 51.32038, 51.32011, + 51.25031, 51.25015, 51.24999, 51.24984, 51.24968, 51.24952, 51.24934, + 51.24919, 51.24903, 51.24885, + 51.26204, 51.26187, 51.26171, 51.26155, 51.26138, 51.26121, 51.26104, + 51.26087, 51.26069, 51.26053, + 51.27377, 51.27361, 51.27343, 51.27325, 51.27308, 51.27291, 51.27274, + 51.27255, 51.27237, 51.27219, + 51.2855, 51.28532, 51.28515, 51.28497, 51.28478, 51.28461, 51.28442, + 51.28424, 51.28405, 51.28386, + 51.29723, 51.29705, 51.29686, 51.29668, 51.29649, 51.2963, 51.29611, + 51.29593, 51.29573, 51.29554, + 51.30896, 51.30877, 51.30858, 51.30839, 51.3082, 51.308, 51.3078, 51.30761, + 51.30741, 51.30721 ; + + Longitude = + 142.8715, 142.8586, 142.8457, 142.8328, 142.82, 142.8071, 142.7943, + 142.7815, 142.7687, 142.7559, + 142.8712, 142.8583, 142.8454, 142.8325, 142.8196, 142.8068, 142.794, + 142.7811, 142.7684, 142.7556, + 142.8709, 142.858, 142.8451, 142.8322, 142.8193, 142.8065, 142.7937, + 142.7808, 142.7681, 142.7553, + 142.8706, 142.8577, 142.8448, 142.8319, 142.819, 142.8062, 142.7933, + 142.7805, 142.7677, 142.7549, + 142.8644, 142.8515, 142.8386, 142.8257, 142.8129, 142.8, 142.7872, + 142.7744, 142.7617, 142.7489, + 142.864, 142.8511, 142.8382, 142.8253, 142.8125, 142.7997, 142.7868, + 142.774, 142.7612, 142.7485, + 142.8637, 142.8508, 142.8379, 142.825, 142.8121, 142.7993, 142.7865, + 142.7736, 142.7608, 142.7481, + 142.8633, 142.8504, 142.8375, 142.8246, 142.8118, 142.7989, 142.7861, + 142.7733, 142.7605, 142.7477, + 142.8629, 142.85, 142.8371, 142.8242, 142.8114, 142.7985, 142.7857, + 142.7729, 142.7601, 142.7473, + 142.8626, 142.8497, 142.8368, 142.8239, 142.811, 142.7981, 142.7853, + 142.7725, 142.7597, 142.7469 ; + + QCAll = + 1, 0, 0, 0, 1, 2, 2, 2, 1, 2, + 2, 1, 2, 0, 1, 0, 1, 2, 1, 2, + 0, 2, 2, 2, 0, 2, 0, 1, 0, 0, + 1, 1, 2, 1, 2, 1, 0, 0, 1, 2, + 2, 0, 1, 1, 0, 1, 2, 1, 0, 2, + 1, 0, 1, 2, 1, 0, 0, 1, 0, 1, + 2, 0, 1, 0, 1, 0, 2, 2, 1, 2, + 0, 0, 1, 1, 2, 1, 1, 2, 0, 0, + 2, 2, 1, 2, 1, 0, 1, 1, 2, 1, + 2, 0, 0, 2, 1, 1, 2, 2, 2, 1 ; + + QCPath = + 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, + 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, + 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, + 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, + 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, + 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, + 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 1, 1, 0 ; + + Residual = + 0.01463089, 0.008259912, 0.0009736047, 0.005178552, 0.0003064495, + 0.01201278, 0.003118906, 0.002148788, 0.01049841, 0.0008692471, + 0.003849114, 0.007654977, 0.01492888, 0.002197735, 0.006742722, + 0.009021605, 0.001459087, 0.004331023, 0.010812, 0.008262091, + 0.01257866, 0.008704971, 0.002768576, 0.009232531, 0.01330433, 0.007751839, + 0.009392155, 0.007575887, 0.01364494, 0.00619957, + 0.00803203, 0.005139647, 0.001943652, 0.009942327, 0.01403505, 0.009190938, + 0.01264108, 0.003268627, 0.01354758, 0.0001465497, + 0.003387191, 0.001980827, 0.01361314, 0.01365346, 0.008715869, 0.001312396, + 0.001753192, 0.01156607, 0.01099193, 0.001306925, + 0.005361948, 0.01159817, 0.001972074, 0.008067487, 0.01132085, 0.004087893, + 0.008497755, 0.007150275, 0.008349311, 0.006611056, + 0.01040606, 0.01077356, 0.01134574, 0.0005598301, 0.01018253, 0.007158312, + 0.001502571, 0.009212944, 0.01256872, 0.01100841, + 0.004823484, 0.00101783, 0.0005599536, 0.008386879, 0.00241279, + 0.004018323, 0.003535679, 0.0002902119, 0.002275296, 0.0005085377, + 0.01472752, 0.00540728, 0.01238197, 0.006183561, 0.003797687, 0.003550663, + 0.01159673, 0.01006885, 0.01059496, 0.01279733, + 0.007830744, 0.006641545, 0.008306227, 0.009779923, 0.01182831, 0.01338404, + 0.004636577, 0.002034147, 0.0112639, 0.007902498 ; +} diff --git a/utils/test/testinput/gdas_viirsaod2ioda.yaml b/utils/test/testinput/gdas_viirsaod2ioda.yaml new file mode 100644 index 000000000..ac99cf019 --- /dev/null +++ b/utils/test/testinput/gdas_viirsaod2ioda.yaml @@ -0,0 +1,11 @@ +provider: VIIRSAOD +window begin: '2018-04-15T03:00:00Z' +window end: '2018-04-15T09:00:00Z' +variable: aerosolOpticalDepth +output file: viirs_out_2018061506.ioda.nc +input files: +- viirs_aod_1.nc4 +- viirs_aod_2.nc4 +thinning: + threshold: 0 +channel: 4 From 041d0e1e33617f65877811d1dd4cb8ab9790ff44 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Dec 2023 10:48:43 -0500 Subject: [PATCH 18/22] Fix example GDAS OOPS utility (#800) Time Window issues strike again! I haven't tested yet because I think fv3-jedi also needs merged now... --- test/land/letkfoi_land.yaml | 5 +++-- utils/ioda_example/gdas_meanioda.h | 10 ++-------- utils/test/testinput/gdas_meanioda.yaml | 7 ++++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/test/land/letkfoi_land.yaml b/test/land/letkfoi_land.yaml index 98cd3d171..635478139 100644 --- a/test/land/letkfoi_land.yaml +++ b/test/land/letkfoi_land.yaml @@ -164,5 +164,6 @@ observations: output increment: filename_sfcd: xainc.sfc_data.nc filetype: fms restart -window begin: '2021-03-23T15:00:00Z' -window length: PT6H +time window: + begin: '2021-03-23T15:00:00Z' + length: PT6H diff --git a/utils/ioda_example/gdas_meanioda.h b/utils/ioda_example/gdas_meanioda.h index 13252c30a..00eeebd51 100644 --- a/utils/ioda_example/gdas_meanioda.h +++ b/utils/ioda_example/gdas_meanioda.h @@ -41,14 +41,8 @@ namespace gdasapp { oops::Log::info() << "obs space: " << std::endl << obsConfig << std::endl; // time window stuff - std::string winbegin; - std::string winend; - fullConfig.get("window begin", winbegin); - fullConfig.get("window end", winend); - bool shift; - fullConfig.get("window shift", shift); - const util::TimeWindow timeWindow(util::DateTime(winbegin), util::DateTime(winend), - util::boolToWindowBound(shift)); + const eckit::LocalConfiguration timeWindowConf(fullConfig, "time window"); + const util::TimeWindow timeWindow(timeWindowConf); // what variable to get the mean of std::string group; diff --git a/utils/test/testinput/gdas_meanioda.yaml b/utils/test/testinput/gdas_meanioda.yaml index fdd0dcd8c..555ca25e7 100644 --- a/utils/test/testinput/gdas_meanioda.yaml +++ b/utils/test/testinput/gdas_meanioda.yaml @@ -1,7 +1,8 @@ # the window is 30 years long to capture anything we can throw at it in this input file -window begin: 2000-11-01T09:00:00Z -window end: 2030-11-01T15:00:00Z -window shift: false +time window: + begin: 2000-11-01T09:00:00Z + end: 2030-11-01T15:00:00Z + bound to include: begin obs space: name: gmi_gpm_test_mean obsdatain: From 0d4551103f700e4f2f6c983f77536d28ec9f10d1 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 11 Dec 2023 12:43:31 -0500 Subject: [PATCH 19/22] Fix UFO nightly testing script for time window change (#802) As the title says. Darth Vader - NOAA Affiliate was not happy over the weekend, this should fix that. --- ush/ufoeval/run_ufo_hofx_test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 230966284..5cb77fb6e 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -33,7 +33,6 @@ run_filtering=YES run_eva=YES eva_stats_only=NO keep_output=NO -window_shift=False while getopts "c:hsxq" opt; do case $opt in @@ -178,13 +177,13 @@ export OPREFIX=gdas.t${cyc}z export APREFIX=gdas.t${cyc}z export GPREFIX=gdas.t${gcyc}z -if [ $obtype == "conv_ps" ]; then window_shift=True; fi cat > $workdir/temp.yaml << EOF -window begin: '{{ WINDOW_BEGIN | to_isotime }}' -window end: '{{ WINDOW_END | to_isotime }}' +time window: + begin: '{{ WINDOW_BEGIN | to_isotime }}' + end: '{{ WINDOW_END | to_isotime }}' + bound to include: begin observations: - !INC $yamlpath -window shift: ${window_shift} EOF $GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml From 1a6ab48318fb8d46bb4e2a4f7ec6a7fc3e83933b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:23:53 -0500 Subject: [PATCH 20/22] Add some error handling to marine obs prep task (#797) Some error handling to the obs prep task: - fails if the `OBS_YAML` file is incorrect - prints a warning and skips to the next observation source if the yaml for that particular source is wrong, or there are no files in `DMPDIR` to copy and convert `print` statements used instead of `logging` due to `logging` not working for some reason --------- Co-authored-by: Guillaume Vernieres --- scripts/exglobal_prep_ocean_obs.py | 51 ++++++++++++++++++++++-------- ush/soca/prep_marine_obs.py | 26 +++++++-------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index ff424557a..e7899ef63 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -8,6 +8,7 @@ import subprocess from wxflow import YAMLFile, save_as_yaml +# TODO (AFE) figure out why logger is not logging # set up logger logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') @@ -24,30 +25,52 @@ OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC') OBS_YAML = os.getenv('OBS_YAML') +# this will fail with FileNotFoundError if all yaml files in OBS_YAML are not +# present in OBS_YAML_DIR obsConfig = YAMLFile(OBS_YAML) OBSPROC_YAML = os.getenv('OBSPROC_YAML') obsprocConfig = YAMLFile(OBSPROC_YAML) # TODO (AFE): needs more error handling (missing sources, missing files) -for observer in obsConfig['observers']: +try: + # For each of the observation sources (observers) specificed in the OBS_YAML... + for observer in obsConfig['observers']: - obsSpaceName = observer['obs space']['name'] - print(f"obsSpaceName: {obsSpaceName}") + try: + obsSpaceName = observer['obs space']['name'] + print(f"obsSpaceName: {obsSpaceName}") + except KeyError: + print(f"observer: {observer}") + print("WARNING: Ill-formed observer yaml file, skipping") + continue # to next observer - for observation in obsprocConfig['observations']: +# ...look through the observations in OBSPROC_YAML... + for observation in obsprocConfig['observations']: - obsprocSpace = observation['obs space'] - obsprocSpaceName = obsprocSpace['name'] + obsprocSpace = observation['obs space'] + obsprocSpaceName = obsprocSpace['name'] - if obsprocSpaceName == obsSpaceName: +# ...for a matching name, and process the observation source + if obsprocSpaceName == obsSpaceName: - matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace) - obsprocSpace['input files'] = matchingFiles - obsprocSpace['window begin'] = windowBegin - obsprocSpace['window end'] = windowEnd + print(f"obsprocSpaceName: {obsSpaceName}") - iodaYamlFilename = obsprocSpaceName + '2ioda.yaml' - save_as_yaml(obsprocSpace, iodaYamlFilename) + # fetch the obs files from DMPDIR to RUNDIR + matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace) - subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True) + if not matchingFiles: + print("WARNING: No files found for obs source , skipping") + break # to next observation source in OBS_YAML + + obsprocSpace['input files'] = matchingFiles + obsprocSpace['window begin'] = windowBegin + obsprocSpace['window end'] = windowEnd + + iodaYamlFilename = obsprocSpaceName + '2ioda.yaml' + save_as_yaml(obsprocSpace, iodaYamlFilename) + + subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True) +except TypeError: + print("CRITICAL: Ill-formed OBS_YAML file, exiting") + raise diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py index 7202d6be9..6bae569fd 100755 --- a/ush/soca/prep_marine_obs.py +++ b/ush/soca/prep_marine_obs.py @@ -8,10 +8,10 @@ DMPDIR = os.getenv('DMPDIR') cyc = os.getenv('cyc') PDY = os.getenv('PDY') -CDUMP = os.getenv('CDUMP') +RUN = os.getenv('RUN') COMIN_OBS = os.getenv('COMIN_OBS') -cycdir = os.path.join(DMPDIR, CDUMP + '.' + str(PDY), str(cyc)) +cycDir = os.path.join(DMPDIR, RUN + '.' + str(PDY), str(cyc)) # TODO: this looks good for a yaml obs_dict = { @@ -65,27 +65,27 @@ def obs_fetch(obsprocSpace): - subdir = obsprocSpace['obsproc subdir'] + subDir = obsprocSpace['obsproc subdir'] filepattern = obsprocSpace['obsproc regex'] - datadir = os.path.join(cycdir, subdir) + dataDir = os.path.join(cycDir, subDir) # TODO: check the existence of this - print('datadir:', datadir) + print('dataDir:', dataDir) matchingFiles = [] - for root, _, files in os.walk(datadir): + for root, _, files in os.walk(dataDir): for filename in fnmatch.filter(files, filepattern): matchingFiles.append(filename) - obs_cpy = [] - for obs_src in matchingFiles: - obs_path = os.path.join(datadir, obs_src) - obs_dst = os.path.join(COMIN_OBS, obs_src) - obs_cpy.append([obs_path, obs_dst]) + obsCopy = [] + for obsSource in matchingFiles: + obsPath = os.path.join(dataDir, obsSource) + obsDestination = os.path.join(COMIN_OBS, obsSource) + obsCopy.append([obsPath, obsDestination]) - print(f"obs_cpy: {obs_cpy}") + print(f"obsCopy: {obsCopy}") print(f"matchingFiles: {matchingFiles}") - FileHandler({'copy': obs_cpy}).sync() + FileHandler({'copy': obsCopy}).sync() return matchingFiles From 066dc816154c6cd8425ccb7b75db586b694492a0 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:47:32 -0500 Subject: [PATCH 21/22] Merge feature/gdas-validation as of 12/5 into develop (#805) Co-authored-by: Emily Liu Co-authored-by: Cory Martin Co-authored-by: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Co-authored-by: BrettHoover-NOAA <98188219+BrettHoover-NOAA@users.noreply.github.com> Co-authored-by: Brett Hoover Co-authored-by: Xuanli Li <101414760+XuanliLi-NOAA@users.noreply.github.com> --- parm/atm/obs/config/atms_n20.yaml | 602 +++++++++----- parm/atm/obs/config/atms_npp.yaml | 557 +++++++++++++ parm/atm/obs/config/gnssro.yaml | 139 ++++ parm/atm/obs/config/iasi_metop-a.yaml | 756 ++++++++++++++++++ parm/atm/obs/config/iasi_metop-b.yaml | 756 ++++++++++++++++++ ..._goes-16.yaml => satwind_abi_goes-16.yaml} | 8 +- ..._goes-17.yaml => satwind_abi_goes-17.yaml} | 8 +- parm/atm/obs/config/satwind_ahi_h8.yaml | 397 +++++++++ parm/atm/obs/config/satwind_seviri_m11.yaml | 405 ++++++++++ parm/atm/obs/config/satwind_seviri_m8.yaml | 405 ++++++++++ ...top-a.yaml => scatwind_ascat_metop-a.yaml} | 18 +- ...top-b.yaml => scatwind_ascat_metop-b.yaml} | 18 +- parm/atm/obs/lists/gdas_prototype_3d.yaml | 15 +- parm/atm/obs/testing/atms_n20.yaml | 251 +++--- parm/atm/obs/testing/atms_n20_noqc.yaml | 148 ++++ parm/atm/obs/testing/atms_npp.yaml | 251 +++--- parm/atm/obs/testing/atms_npp_noqc.yaml | 148 ++++ parm/atm/obs/testing/iasi_metop-a.yaml | 22 +- parm/atm/obs/testing/iasi_metop-a_noqc.yaml | 158 ++++ parm/atm/obs/testing/iasi_metop-b.yaml | 22 +- parm/atm/obs/testing/iasi_metop-b_noqc.yaml | 158 ++++ parm/ioda/bufr2ioda/atms_beamwidth.txt | 36 + .../bufr2ioda_atms.yaml} | 11 +- .../ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json | 34 + parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml | 238 ++++++ .../bufr2ioda/bufr2ioda_satwind_amv_ahi.json | 15 + .../bufr2ioda_satwind_amv_seviri.json | 16 + ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py | 650 +++++++++++++++ .../bufr2ioda/bufr2ioda_satwind_amv_ahi.py | 468 +++++++++++ .../bufr2ioda/bufr2ioda_satwind_amv_goes.py | 20 +- .../bufr2ioda/bufr2ioda_satwind_amv_seviri.py | 441 ++++++++++ ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py | 22 +- ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py | 60 +- ush/ioda/bufr2ioda/run_bufr2ioda.py | 77 +- 34 files changed, 6784 insertions(+), 546 deletions(-) create mode 100644 parm/atm/obs/config/atms_npp.yaml create mode 100644 parm/atm/obs/config/gnssro.yaml create mode 100644 parm/atm/obs/config/iasi_metop-a.yaml create mode 100644 parm/atm/obs/config/iasi_metop-b.yaml rename parm/atm/obs/config/{satwind_goes-16.yaml => satwind_abi_goes-16.yaml} (99%) rename parm/atm/obs/config/{satwind_goes-17.yaml => satwind_abi_goes-17.yaml} (99%) create mode 100644 parm/atm/obs/config/satwind_ahi_h8.yaml create mode 100644 parm/atm/obs/config/satwind_seviri_m11.yaml create mode 100644 parm/atm/obs/config/satwind_seviri_m8.yaml rename parm/atm/obs/config/{scatwind_metop-a.yaml => scatwind_ascat_metop-a.yaml} (94%) rename parm/atm/obs/config/{scatwind_metop-b.yaml => scatwind_ascat_metop-b.yaml} (94%) create mode 100644 parm/atm/obs/testing/atms_n20_noqc.yaml create mode 100644 parm/atm/obs/testing/atms_npp_noqc.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-a_noqc.yaml create mode 100644 parm/atm/obs/testing/iasi_metop-b_noqc.yaml create mode 100644 parm/ioda/bufr2ioda/atms_beamwidth.txt rename parm/ioda/{bufr_atms.yaml => bufr2ioda/bufr2ioda_atms.yaml} (93%) create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json create mode 100755 parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py create mode 100755 ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 28b19e692..5cb9c27f3 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -3,24 +3,29 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_atms_n20_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 - simulated variables: [brightness_temperature] + simulated variables: [brightnessTemperature] channels: &atms_n20_channels 1-22 + obs operator: name: CRTM - Absorbers: [H2O,O3] + Absorbers: [H2O, O3, CO2] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: - Sensor_ID: atms_n20 + Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] + obs bias: input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4 @@ -51,24 +56,141 @@ obs bias: ratio: 1.1 ratio for small dataset: 2.0 output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4 -obs filters: -- filter: BlackList + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - action: - name: assign error - error function: + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorModelRamp@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] + name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, @@ -81,293 +203,355 @@ obs filters: 0.500, 0.500] err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] -# CLW Retrieval Check + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_n20_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_n20_channels + - name: brightnessTemperature@HofX + channels: *atms_n20_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: 1-7, 16-22 + - name: brightnessTemperature + channels: 1-7, 16-22 test variables: - - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + - name: CLWRetFromObs@DerivedMetaData maxvalue: 999.0 - action: - name: reject -# CLW Retrieval Check + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: 1-7, 16-22 + - name: brightnessTemperature + channels: 1-7, 16-22 test variables: - - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + - name: CLWRetFromBkg@DerivedMetaData maxvalue: 999.0 - action: - name: reject -# Hydrometeor Check (cloud/precipitation affected chanels) + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - - name: brightness_temperature - channels: *atms_n20_channels + - name: brightnessTemperature + channels: 1-7, 16, 17-22 test variables: - - name: HydrometeorCheckATMS@ObsFunction + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS channels: *atms_n20_channels - options: + type: float + function: + name: HydrometeorCheckATMS@ObsFunction channels: *atms_n20_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - obserr_function: - name: ObsErrorModelRamp@ObsFunction + options: channels: *atms_n20_channels - options: + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - maxvalue: 0.0 - action: - name: reject -# Topography check -- filter: BlackList + +- filter: Bounds Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - action: - name: inflate error - inflation variable: + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorTopoRad@ObsFunction channels: *atms_n20_channels options: - sensor: atms_n20 + sensor: *Sensor_ID channels: *atms_n20_channels -# Transmittnace Top Check -- filter: BlackList + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorTransmitTopRad@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels -# Surface Jacobian check -- filter: BlackList + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorSurfJacobianRad@ObsFunction channels: *atms_n20_channels options: + sensor: *Sensor_ID channels: *atms_n20_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Situation dependent Check -- filter: BlackList + + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels action: name: inflate error inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + type: float + function: name: ObsErrorFactorSituDependMW@ObsFunction channels: *atms_n20_channels options: - sensor: atms_n20 + sensor: *Sensor_ID channels: *atms_n20_channels - clwobs_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] clwbkg_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData scatobs_function: - name: SCATRetMW@ObsFunction - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] + name: SIRetFromObs@DerivedMetaData clwmatchidx_function: - name: CLWMatchIndexMW@ObsFunction + name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - options: - channels: *atms_n20_channels - clwobs_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] -# Gross check -- filter: Background Check + +- filter: Perform Action filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels - function absolute threshold: - - name: ObsErrorBoundMW@ObsFunction + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData channels: *atms_n20_channels - options: - sensor: atms_n20 + type: float + function: + name: ObsErrorBoundMW@ObsFunction channels: *atms_n20_channels - obserr_bound_latitude: - name: ObsErrorFactorLatRad@ObsFunction - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsErrorFactorTransmitTopRad@ObsFunction + options: + sensor: *Sensor_ID channels: *atms_n20_channels - options: + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *atms_n20_channels - obserr_bound_topo: - name: ObsErrorFactorTopoRad@ObsFunction - channels: *atms_n20_channels - options: + options: + channels: *atms_n20_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData channels: *atms_n20_channels - sensor: atms_n20 - obserr_function: - name: ObsErrorModelRamp@ObsFunction - channels: *atms_n20_channels - options: + obserr_function: + name: InitialObsError@DerivedMetaData channels: *atms_n20_channels - xvar: - name: CLWRetSymmetricMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 2.0, 4.5, - 4.5, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0] - action: - name: reject -# Inter-channel check + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_n20_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels test variables: - name: InterChannelConsistencyCheck@ObsFunction channels: *atms_n20_channels options: channels: *atms_n20_channels - sensor: atms_n20 + use passive_bc: true + sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 - action: - name: reject -# Useflag check + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check - filter: Bounds Check filter variables: - - name: brightness_temperature + - name: brightnessTemperature channels: *atms_n20_channels test variables: - - name: ChannelUseflagCheckRad@ObsFunction + - name: ObsFunction/ChannelUseflagCheckRad channels: *atms_n20_channels options: channels: *atms_n20_channels use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] minvalue: 1.0e-12 - action: - name: reject + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml new file mode 100644 index 000000000..0a9b633b6 --- /dev/null +++ b/parm/atm/obs/config/atms_npp.yaml @@ -0,0 +1,557 @@ +obs space: + name: atms_npp + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)atms_npp.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_atms_npp_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &atms_npp_channels 1-22 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O, O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_npp.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlapse $(DATA)/obs/$(GPREFIX)atms_npp.tlapse.txt + - name: lapse_rate + tlapse: *atms_npp_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)atms_npp.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)atms_npp.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Calculate Innovation@DerivedMetaData +- filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_npp_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_npp_channels + - name: brightnessTemperature@HofX + channels: *atms_npp_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_npp_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *atms_npp_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_npp_channels + +# Step 10: Gross check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *atms_npp_channels + options: + sensor: *Sensor_ID + channels: *atms_npp_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_npp_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_npp_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/gnssro.yaml b/parm/atm/obs/config/gnssro.yaml new file mode 100644 index 000000000..949a4f87b --- /dev/null +++ b/parm/atm/obs/config/gnssro.yaml @@ -0,0 +1,139 @@ +obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + sr_steps: 2 + vertlayer: full + super_ref_qc: NBAM +obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)gnssro.tm00.bufr_d.nc +# obsgrouping: +# group variables: [ 'sequenceNumber' ] +# sort variable: 'impactHeightRO' +# sort order: 'ascending' + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_gnssro_{{ current_cycle | to_YMDH }}.nc4 + simulated variables: [bendingAngle] + +obs filters: +# Apply gross check using pccf +# Step 0-A: Create Diagnostic Flags +# Diagnostic flag for pccf +- filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: pccfCheckReject + initial value: false + force reinitialization: true + +# Diagnostic flag for qfro +- filter: Create Diagnostic Flags + filter variables: + - name: bendingAngle + flags: + - name: qfroCheckReject + initial value: false + force reinitialization: true + +# Step 0-B: pccf Check - good: 0.1-100, reject: 0 +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265-269,750-755,44,786,820,825 + test variables: + - name: MetaData/pccf + minvalue: 0.1 + maxvalue: 100.1 + actions: + - name: set + flag: pccfCheckReject + - name: reject + +# Step 0-B: qfro Check - good: 0, reject: 1 +- filter: Bounds Check +- filter: RejectList + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5,421,440,821 + test variables: + - name: MetaData/qualityFlags + minvalue: -0.1 + maxvalue: 0.1 + actions: + - name: set + flag: qfroCheckReject + - name: reject + +#1. gpstop +- filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject +#2. commgpstop +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject +#3. metop below 8 km +- filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject +#4. assign obs error +- filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM +#5. RONBAM cut off check +- filter: Background Check RONBAM + filter variables: + - name: bendingAngle +#6. Obs error inflate +- filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate +#7. Background check +#- filter: Background Check +# filter variables: +# - name: bendingAngle +# threshold: 10 +# action: +# name: reject + diff --git a/parm/atm/obs/config/iasi_metop-a.yaml b/parm/atm/obs/config/iasi_metop-a.yaml new file mode 100644 index 000000000..f5c7a602a --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-a.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-a.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-a_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-a.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-a.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-a.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-a_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-a_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-a_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-a_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-a_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: iasi_metop-a + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-a_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-a + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-a_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/config/iasi_metop-b.yaml b/parm/atm/obs/config/iasi_metop-b.yaml new file mode 100644 index 000000000..1a4681dc6 --- /dev/null +++ b/parm/atm/obs/config/iasi_metop-b.yaml @@ -0,0 +1,756 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)mtiasi_metop-b.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_iasi_metop-b_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + observed variables: [radiance] + simulated variables: [brightnessTemperature] + derived variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlapse $(DATA)/obs/$(GPREFIX)iasi_metop-b.tlapse.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)iasi_metop-b.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)iasi_metop-b.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +#- filter: Create Diagnostic Flags +# filter variables: +# - name: brightnessTemperature +# channels: *iasi_metop-b_channels +# flags: +# - name: ScanEdgeRemoval +# initial value: false +# force reinitialization: false +# - name: Thinning +# initial value: false +# force reinitialization: false +# - name: ShortwaveIRCheck +# initial value: false +# force reinitialization: false +# - name: ObsValueRangeCheck +# initial value: false +# force reinitialization: false +# - name: CloudDetection +# initial value: false +# force reinitialization: false +# - name: NSSTCheck +# initial value: false +# force reinitialization: false +# - name: GrossCheck +# initial value: false +# force reinitialization: false +# - name: InterChannelConsistency +# initial value: false +# force reinitialization: false +# - name: UseFlagCheck +# initial value: false +# force reinitialization: false + +# Step 0-B: Create Derived Variables +# Assign channel wavenumbers in m-1 +- filter: Variable Assignment + assignments: + - name: MetaData/sensorCentralWavenumber + type: float + channels: *iasi_metop-b_channels + function: + name: ObsFunction/LinearCombination + options: + variables: + - name: ObsValue/radiance + channels: *iasi_metop-b_channels + coefs: [25.0] + intercept: 64475.0 + use channel numbers: true + +# Transform radiance to brightness temperature +- filter: Variable Transforms + Transform: SatBrightnessTempFromRad + transform from: + name: ObsValue/radiance + channels: *iasi_metop-b_channels + spectral variable: + name: MetaData/sensorCentralWavenumber + channels: *iasi_metop-b_channels + radiance units: wavenumber + planck1: 1.191042953e-16 + planck2: 1.4387774e-2 + +# Step 0-C: Assign Observation Error +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] + +obs post filters: +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 5-56 + action: + name: reject +# actions: +# - name: set +# flag: ScanEdgeRemoval +# - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: MetaData/fractionOfClearPixelsInFOV +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + action: + name: reject +# actions: +# - name: set +# flag: Thinning +# - name: reject + +# Step 3: Wavenumber Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: water_area_fraction@GeoVaLs + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: ShortwaveIRCheck +# - name: reject + +# Step 4: Observation Error Inflation based on Wavenumber +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorWavenumIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 5: Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + minvalue: 50.00001 + maxvalue: 449.99999 +# maxvalue: 100.00000 + action: + name: reject +# actions: +# - name: set +# flag: ObsValueRangeCheck +# - name: reject + +# Step 6: Topography Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + sensor: iasi_metop-b + +# Step 7: Transmittance Top Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + +# Step 8: Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + error parameter vector: *iasi_metop-b_oberr + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: CloudDetection +# - name: reject + +# Step 9: NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: NearSSTRetCheckIR@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: NSSTCheck +# - name: reject + +# Step 10: Surface Jacobians Check +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] + sensor: iasi_metop-b + +# Step 11: Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, + 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, + 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, + 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, + 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, + 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, + 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, + 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, + 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, + 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, + 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, + 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, + 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, + 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + error parameter vector: *iasi_metop-b_oberr + action: + name: reject +# actions: +# - name: set +# flag: GrossCheck +# - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, + 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, + -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, + 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + minvalue: 1.0e-12 + action: + name: reject +# actions: +# - name: set +# flag: UseFlagCheck +# - name: reject diff --git a/parm/atm/obs/config/satwind_goes-16.yaml b/parm/atm/obs/config/satwind_abi_goes-16.yaml similarity index 99% rename from parm/atm/obs/config/satwind_goes-16.yaml rename to parm/atm/obs/config/satwind_abi_goes-16.yaml index 33442b599..c35f9626b 100644 --- a/parm/atm/obs/config/satwind_goes-16.yaml +++ b/parm/atm/obs/config/satwind_abi_goes-16.yaml @@ -7,7 +7,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_satwind_abi_goes-16_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -17,6 +17,9 @@ obs operator: hofx scaling field: SurfaceWindScalingPressure hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + obs prior filters: # Apply variable changes needed for wind scaling # For wind observations with pressure provided @@ -181,7 +184,7 @@ obs post filters: - name: windEastward - name: windNorthward test variables: - - name: MetaData/sensorZenithAngle + - name: MetaData/satelliteZenithAngle maxvalue: 68. action: name: reject @@ -427,7 +430,6 @@ obs post filters: action: name: reject -obs post filters: # Reject GOES (247) when difference of wind direction is more than 50 degrees. # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - filter: Bounds Check diff --git a/parm/atm/obs/config/satwind_goes-17.yaml b/parm/atm/obs/config/satwind_abi_goes-17.yaml similarity index 99% rename from parm/atm/obs/config/satwind_goes-17.yaml rename to parm/atm/obs/config/satwind_abi_goes-17.yaml index b413b969b..19eee100a 100644 --- a/parm/atm/obs/config/satwind_goes-17.yaml +++ b/parm/atm/obs/config/satwind_abi_goes-17.yaml @@ -7,7 +7,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_satwind_abi_goes-17_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -17,6 +17,9 @@ obs operator: hofx scaling field: SurfaceWindScalingPressure hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + obs prior filters: # Apply variable changes needed for wind scaling # For wind observations with pressure provided @@ -181,7 +184,7 @@ obs post filters: - name: windEastward - name: windNorthward test variables: - - name: MetaData/sensorZenithAngle + - name: MetaData/satelliteZenithAngle maxvalue: 68. action: name: reject @@ -427,7 +430,6 @@ obs post filters: action: name: reject -obs post filters: # Reject GOES (247) when difference of wind direction is more than 50 degrees. # CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged - filter: Bounds Check diff --git a/parm/atm/obs/config/satwind_ahi_h8.yaml b/parm/atm/obs/config/satwind_ahi_h8.yaml new file mode 100644 index 000000000..c66afaa67 --- /dev/null +++ b/parm/atm/obs/config/satwind_ahi_h8.yaml @@ -0,0 +1,397 @@ +obs space: + name: satwind_ahi_h8 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.ahi_h8.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_ahi_h8_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of AHI/Himawari-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #741 for details: https://github.com/NOAA-EMC/GDASApp/issues/741 +#obs pre filters: +#- filter: Gaussian Thinning +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML +# Type 242 (Himawari VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# Type 250 (Himawari AHI WV, cloud-top or clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,7.,7.3,7.6,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.,8.] + + +# Type Type 252 (Himawari AHI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 252 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 250 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 250 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. (also > 100., which is missing data) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# Reject Type 252 (IR) winds with a /=0 surface type (non-water surface) when latitude > 20. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: ObsType/windEastward + is_in: 252 + test variables: + - name: MetaData/latitude + maxvalue: 20. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# IR (Type 242), reject when pressure is less than 700 mb +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 242 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# cloud-top WV (Type 250), reject when pressure greater than 399 mb. +# CLEARED: maxvalue is rejecting >, not >= as per a Perform Action, so threshold is unchanged +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 250 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# JMA IR (252) reject when pressure between 499 and 801 mb. +# PERFORM ACTION: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 49901. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 252 + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/satwind_seviri_m11.yaml b/parm/atm/obs/config/satwind_seviri_m11.yaml new file mode 100644 index 000000000..763e59fda --- /dev/null +++ b/parm/atm/obs/config/satwind_seviri_m11.yaml @@ -0,0 +1,405 @@ +obs space: + name: satwind_seviri_m11 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.seviri_m11.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_seviri_m11_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of SEVIRI/METEOSAT-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #758 for details: https://github.com/NOAA-EMC/GDASApp/issues/758 +#obs pre filters: +#- filter: Gaussian Thinning +# where: +# - variable: ObsType/windEastward +# is_in: 243, 253 +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 254 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 254 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + + +# Exclude data over non-water surface type where latitude > 20N for Type 253 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# EUMET VIS (243) reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# EUMET WV (254) reject when pressure greater than 399 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/satwind_seviri_m8.yaml b/parm/atm/obs/config/satwind_seviri_m8.yaml new file mode 100644 index 000000000..70ada0f9d --- /dev/null +++ b/parm/atm/obs/config/satwind_seviri_m8.yaml @@ -0,0 +1,405 @@ +obs space: + name: satwind_seviri_m8 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)satwnd.seviri_m8.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_satwind_seviri_m8_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [windEastward, windNorthward] + +obs operator: + name: VertInterp + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + +linear obs operator: + name: VertInterp + +# NOTE: Tests using the Gaussian Thinning filter (below) to duplicate GSI's thinning of SEVIRI/METEOSAT-8 satwinds +# results in more JEDI satwinds in the diag file than in GSI, but far fewer JEDI satwinds assimilated than +# GSI. JEDI under-counts assimilated winds by roughly 25-40%, relative to GSI, and this under-count is not +# even including the temporal thinning which is applied in GSI but not JEDI (by this filter below). See +# GDASApp Issue #758 for details: https://github.com/NOAA-EMC/GDASApp/issues/758 +#obs pre filters: +#- filter: Gaussian Thinning +# where: +# - variable: ObsType/windEastward +# is_in: 243, 253 +# horizontal_mesh: 200 +# vertical_mesh: 10000 +# use_reduced_horizontal_grid: true +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + +obs prior filters: +# Apply variable changes needed for wind scaling +# For wind observations with pressure provided +- filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: False + +# Calculate error inflation factor for duplicate observations +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windEastward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windEastward + +#- filter: Variable Assignment +# assignments: +# - name: ObsErrorFactorDuplicateCheck/windNorthward +# type: float +# function: +# name: ObsFunction/ObsErrorFactorDuplicateCheck +# options: +# use_air_pressure: true +# variable: windNorthward + +obs post filters: +# Assign the initial observation error, based on height/pressure +# Hard-wiring to prepobs_errtable.global by Type +# ObsError is currently not updating in diag file, but passes directly to EffectiveError when no inflation is specified in YAML + +# Type 243 (MVIRI/SEVIRI VIS) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 253 (MVIRI/SEVERI LWIR) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.,4.1,5.,6.,6.3,6.6,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + +# Type 254 (MVIRI/SEVIRI WV, both cloud-top and clear-sky) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + minvalue: -135. + maxvalue: 135. + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000.,105000.,100000.,95000.,90000.,85000.,80000.,75000.,70000.,65000.,60000.,55000.,50000.,45000.,40000.,35000.,30000.,25000.,20000.,15000.,10000.,7500.,5000.,4000.,3000.,2000.,1000.,500.,400.,300.,200.,100.,0.] #Pressure (Pa) + errors: [3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.8,3.9,3.9,4.,4.5,6.1,6.,6.5,7.3,7.6,7.,7.5,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.,7.] + + +# sanity-check criteria +# Observation Range Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -130. + maxvalue: 130. + action: + name: reject + +# Velocity Sanity Check +# NOT EXPLICITLY CLEARED: No obs in this range in file, so 0 Bounds Check rejects (which is correct) but essentially untested +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Velocity + maxvalue: 130. + action: + name: reject + +# GSI read routine QC (part-1) +# Exclude Type 254 with windComputationMethod==5 (clear-sky WV) --- obs tossed without passing to setup routine +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 254 + - variable: MetaData/windComputationMethod + is_in: 5 + action: + name: reject + +# Exclude data with satellite zenith angle > 68 for all types --- obs tossed without passing to setup routine +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/satelliteZenithAngle + maxvalue: 68. + action: + name: reject + + +# Exclude data over non-water surface type where latitude > 20N for Type 253 (IRLW) --- obs tossed and not passed to setup routine +# Notes: This check was missing, so added (eliu) +# Replace land_type_index_NPOSS with water_area_fraction (eliu) +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 253 + - variable: MetaData/latitude + minvalue: 20. + test variables: + - name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# GSI read routine QC (part-2) +# Reject obs with qualityInformationWithoutForecast < 85. OR > 100. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: MetaData/qualityInformationWithoutForecast + minvalue: 85. + maxvalue: 100. + action: + name: reject + +# GSI setupw routine QC +# Reject any ob Type [240–260] when pressure greater than 950 mb. +# CLEARED: minvalue/maxvalue are >=/<=, not >/<, so editing range by 1 Pa +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 240-260 + test variables: + - name: MetaData/pressure + maxvalue: 95001. + action: + name: reject + +# EUMETSAT IR (253) reject when pressure between 401 and 801 mb. +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: MetaData/pressure + minvalue: 40101. + maxvalue: 80099. + - variable: ObsType/windEastward + is_in: 253 + action: + name: reject + +# EUMET VIS (243) reject when pressure less than 700 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 243 + test variables: + - name: MetaData/pressure + minvalue: 70000. + action: + name: reject + +# EUMET WV (254) reject when pressure greater than 399 mb. +- filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: 254 + test variables: + - name: MetaData/pressure + maxvalue: 39900. + action: + name: reject + +# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. +# CLEARED: minvalue is rejecting <, not <= as per a Perform Action, so threshold is unchanged +# Notes (eliu): This tropopause check reject too many obs; probably due to tropopause pressure estimation +# Turn this check off for now. +# Need to check if troposphere pressure was implemented correctly in fv3-jed +- filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/tropopause_pressure + value: MetaData/pressure + minvalue: -5000. # 50 hPa above tropopause level, negative p-diff + action: + name: reject + +# All satwinds must adjust errors based on ObsErrorFactorPressureCheck +# prior to the SPDB check (i.e. the gross-error check). The gross-error +# check uses the adjusted errors for error-bound tightening and rejection, +# so this check has to come first. This check will inflate errors for obs +# that are too close to either the model top or bottom. +# Notes (eliu): GMAO added a required parameter: adjusted_error_name. +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + surface_obs: false + variable: windEastward + inflation factor: 4.0 + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + +# All satwinds subject to a gross-error check that contains significant +# modifiers for satwinds with a negative speed-bias. ALL wind gross-error +# checks are currently being done by the SatWindsSPDBCheck. +# CLEARED +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windEastward + action: + name: reject + +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 1.3, 1.3, 2.5, 2.5, 2.5, 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1, 20.1] + variable: windNorthward + action: + name: reject + +# The last error inflation check is for duplicate observations. This one needs +# to come last, because we don't want to inflate errors for duplication if one +# of the duplicates should be rejected. +# Notes (eliu): ObsErrorFactorDuplicateCheck obsfunction requires PreUseFlag (usage parameter from read_satwnd.f90). +# : Turn off duplicate check for now. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windEastward + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# action: +# name: inflate error +# inflation variable: +# name: ObsErrorFactorDuplicateCheck/windNorthward + +# We are extending this to an additional filter that inflates final ob-errors across-the-board by +# 1/0.8 = 1.25. This is caused by the GSI value of nvqc being set to .true. in the global operational +# configuration, see: https://github.com/NOAA-EMC/global-workflow/blob/d5ae3328fa4041b177357b1133f6b92e81c859d7/scripts/exglobal_atmos_analysis.sh#L750 +# This setting activates Line 1229 of setupw.f90 to scale ratio_errors by 0.8, which is applied in +# the denominator of the final ob-error, so 1/0.8 = 1.25 factor of ob-error inflation. +# +# If this nvqc functionality were to be switched off (i.e. if variational qc were to be turned off), +# you would want to remove this last inflation filter. +#- filter: Perform Action +# filter variables: +# - name: windEastward +# where: +# - variable: ObsType/windEastward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +#- filter: Perform Action +# filter variables: +# - name: windNorthward +# where: +# - variable: ObsType/windNorthward +# is_in: 240-260 +# action: +# name: inflate error +# inflation factor: 1.25 + +# End of Filters diff --git a/parm/atm/obs/config/scatwind_metop-a.yaml b/parm/atm/obs/config/scatwind_ascat_metop-a.yaml similarity index 94% rename from parm/atm/obs/config/scatwind_metop-a.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-a.yaml index 4735c31fe..9dcbe9eab 100644 --- a/parm/atm/obs/config/scatwind_metop-a.yaml +++ b/parm/atm/obs/config/scatwind_ascat_metop-a.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-a.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-a_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -23,6 +23,20 @@ obs operator: hofx scaling field: SurfaceWindScalingHeight hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + +obs pre filters: +- filter: Gaussian Thinning + horizontal_mesh: 75 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + obs prior filters: # Apply variable changes needed for rescaled height coordinate - filter: Variable Transforms diff --git a/parm/atm/obs/config/scatwind_metop-b.yaml b/parm/atm/obs/config/scatwind_ascat_metop-b.yaml similarity index 94% rename from parm/atm/obs/config/scatwind_metop-b.yaml rename to parm/atm/obs/config/scatwind_ascat_metop-b.yaml index 5ed7c38bc..16f10351b 100644 --- a/parm/atm/obs/config/scatwind_metop-b.yaml +++ b/parm/atm/obs/config/scatwind_ascat_metop-b.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ascatw.ascat_metop-b.tm00.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_ascatw_ascat_metop-b_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] @@ -23,6 +23,20 @@ obs operator: hofx scaling field: SurfaceWindScalingHeight hofx scaling field group: DerivedVariables +linear obs operator: + name: VertInterp + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + +obs pre filters: +- filter: Gaussian Thinning + horizontal_mesh: 75 + use_reduced_horizontal_grid: true + round_horizontal_bin_count_to_nearest: true + partition_longitude_bins_using_mesh: true + obs prior filters: # Apply variable changes needed for rescaled height coordinate - filter: Variable Transforms diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index 18d64e54e..b32d4b634 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -1,13 +1,20 @@ observers: -- !INC ${OBS_YAML_DIR}/amsua_n19.yaml -- !INC ${OBS_YAML_DIR}/sondes.yaml +##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml +##- !INC ${OBS_YAML_DIR}/sondes_prepbufr.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml +##- !INC ${OBS_YAML_DIR}/atms_npp.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml -##- !INC ${OBS_YAML_DIR}/satwind.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-16.yaml +##- !INC ${OBS_YAML_DIR}/satwind_abi_goes-17.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/scatwind_ascat_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/omi_aura.yaml ##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml ##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml ##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-a.yaml +##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml -#- !INC ${OBS_YAML_DIR}/sfcship.yaml +##- !INC ${OBS_YAML_DIR}/sfcship.yaml +##- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/atm/obs/testing/atms_n20.yaml b/parm/atm/obs/testing/atms_n20.yaml index 1eaf198ce..006a452f3 100644 --- a/parm/atm/obs/testing/atms_n20.yaml +++ b/parm/atm/obs/testing/atms_n20.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_n20 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_n20 obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV atms_n20_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-22 + channels: &atms_n20_channels 1-22 + geovals: filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_n20_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck initial value: false - force reinitialization: true + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -153,13 +146,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: CLWMatchIndexMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels clwobs_function: name: CLWRetFromObs@DerivedMetaData clwbkg_function: @@ -174,13 +167,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorModelRamp@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels xvar: name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, @@ -208,29 +201,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: Innovation@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *atms_n20_channels options: variables: - name: brightnessTemperature@ObsValue - channels: *all_channels + channels: *atms_n20_channels - name: brightnessTemperature@HofX - channels: *all_channels + channels: *atms_n20_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: assign error error function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,31 +262,44 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels test variables: - - name: HydrometeorCheckATMS@ObsFunction - channels: *all_channels - options: - channels: *all_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *all_channels + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -301,59 +307,59 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorTopoRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] @@ -361,25 +367,25 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorFactorSituDependMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels clwbkg_function: name: CLWRetFromBkg@DerivedMetaData clwobs_function: @@ -388,25 +394,25 @@ obs post filters: name: SIRetFromObs@DerivedMetaData clwmatchidx_function: name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels action: name: inflate error inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -422,44 +428,43 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels type: float function: name: ObsErrorBoundMW@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_n20_channels obserr_bound_latitude: name: ObsErrorFactorLat@DerivedMetaData obserr_bound_transmittop: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels obserr_bound_topo: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels function absolute threshold: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_n20_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -467,25 +472,47 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_n20_channels test variables: - name: InterChannelConsistencyCheck@ObsFunction - channels: *all_channels + channels: *atms_n20_channels options: - channels: *all_channels + channels: *atms_n20_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck ignore: rejected observations - name: reject -passedBenchmark: 171071 # GSI: 171076 -# Notes: one point difference in HydrometeorCheck for channel 7 + +passedBenchmark: 181401 # GSI: 181403 (difference is in channel 7) diff --git a/parm/atm/obs/testing/atms_n20_noqc.yaml b/parm/atm/obs/testing/atms_n20_noqc.yaml new file mode 100644 index 000000000..95c2463a9 --- /dev/null +++ b/parm/atm/obs/testing/atms_n20_noqc.yaml @@ -0,0 +1,148 @@ +obs space: + name: atms_n20 + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_n20_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_n20_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &atms_n20_channels 1-22 + +geovals: + filename: !ENV atms_n20_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_n20 + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV atms_n20_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_n20_tlap !ENV atms_n20_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_n20_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + +passedBenchmark: 249449 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/atms_npp.yaml b/parm/atm/obs/testing/atms_npp.yaml index 6558d79cc..c83fc7b8f 100644 --- a/parm/atm/obs/testing/atms_npp.yaml +++ b/parm/atm/obs/testing/atms_npp.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID atms_npp - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: atms_npp obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV atms_npp_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-22 + channels: &atms_npp_channels 1-22 + geovals: filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV atms_npp_satbias_${GDATE}.nc4 variational bc: @@ -42,45 +45,35 @@ obs bias: obs post filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels flags: - - name: CLWRetrievalReject + - name: ScanEdgeRemoval initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject + force reinitialization: false + - name: Thinning initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject + force reinitialization: false + - name: CLWRetrievalCheck initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: InterChannelCheckReject + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck initial value: false - force reinitialization: true + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false # Step 0-B: Calculate derived variables # Calculate CLW retrieved from observation @@ -153,13 +146,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: CLWMatchIndexMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels clwobs_function: name: CLWRetFromObs@DerivedMetaData clwbkg_function: @@ -174,13 +167,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorModelRamp@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels xvar: name: CLWRetSymmetric@DerivedMetaData x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, @@ -208,29 +201,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: Innovation@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *atms_npp_channels options: variables: - name: brightnessTemperature@ObsValue - channels: *all_channels + channels: *atms_npp_channels - name: brightnessTemperature@HofX - channels: *all_channels + channels: *atms_npp_channels coefs: [1, -1] # Step 1: Assign initial all-sky observation error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: assign error error function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -242,7 +235,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -255,7 +248,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 4: Window channel sanity check @@ -269,31 +262,44 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels test variables: - - name: HydrometeorCheckATMS@ObsFunction - channels: *all_channels - options: - channels: *all_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *all_channels + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_npp_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject @@ -301,59 +307,59 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorTopoRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 7: Obs error inflation based on TOA transmittancec check - filter: Variable Assignment assignments: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 8: Observation error inflation based on surface jacobian check - filter: Variable Assignment assignments: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] @@ -361,25 +367,25 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 9: Situation dependent check - filter: Variable Assignment assignments: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorFactorSituDependMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels clwbkg_function: name: CLWRetFromBkg@DerivedMetaData clwobs_function: @@ -388,25 +394,25 @@ obs post filters: name: SIRetFromObs@DerivedMetaData clwmatchidx_function: name: CLWMatchIndex@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels action: name: inflate error inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threhold @@ -422,44 +428,43 @@ obs post filters: - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels type: float function: name: ObsErrorBoundMW@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *atms_npp_channels obserr_bound_latitude: name: ObsErrorFactorLat@DerivedMetaData obserr_bound_transmittop: name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels obserr_bound_topo: name: ObsErrorFactorTopo@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels obserr_function: name: InitialObsError@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels function absolute threshold: - name: ObsErrorBound@DerivedMetaData - channels: *all_channels + channels: *atms_npp_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject @@ -467,25 +472,47 @@ obs post filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *atms_npp_channels test variables: - name: InterChannelConsistencyCheck@ObsFunction - channels: *all_channels + channels: *atms_npp_channels options: - channels: *all_channels + channels: *atms_npp_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck ignore: rejected observations - name: reject -passedBenchmark: 169997 # GSI: 169998 -# Notes: one point difference in HydrometeorCheck for channel 7 + +passedBenchmark: 182751 # GSI: 182751 diff --git a/parm/atm/obs/testing/atms_npp_noqc.yaml b/parm/atm/obs/testing/atms_npp_noqc.yaml new file mode 100644 index 000000000..5bbf45307 --- /dev/null +++ b/parm/atm/obs/testing/atms_npp_noqc.yaml @@ -0,0 +1,148 @@ +obs space: + name: atms_npp + obsdatain: + engine: + type: H5File + obsfile: !ENV atms_npp_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV atms_npp_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &atms_npp_channels 1-22 + +geovals: + filename: !ENV atms_npp_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID atms_npp + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV atms_npp_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &atms_npp_tlap !ENV atms_npp_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *atms_npp_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_npp_channels + options: + channels: *atms_npp_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_npp_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_npp_channels + +passedBenchmark: 247947 +#vector ref: GsiHofXBc +##tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/iasi_metop-a.yaml b/parm/atm/obs/testing/iasi_metop-a.yaml index fed8883c4..477c1c871 100644 --- a/parm/atm/obs/testing/iasi_metop-a.yaml +++ b/parm/atm/obs/testing/iasi_metop-a.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-a_channels sensor: iasi_metop-a + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-a_channels options: channels: *iasi_metop-a_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-a_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-a + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-a_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 127348 - +passedBenchmark: 144704 #127348 diff --git a/parm/atm/obs/testing/iasi_metop-a_noqc.yaml b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml new file mode 100644 index 000000000..c9ee251a4 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-a_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-a + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-a_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-a_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-a + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlap iasi_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: &iasi_metop-a_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 801416 diff --git a/parm/atm/obs/testing/iasi_metop-b.yaml b/parm/atm/obs/testing/iasi_metop-b.yaml index 2d0654822..660aae464 100644 --- a/parm/atm/obs/testing/iasi_metop-b.yaml +++ b/parm/atm/obs/testing/iasi_metop-b.yaml @@ -158,7 +158,7 @@ obs prior filters: obs post filters: # Wavenumber Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, @@ -176,7 +176,8 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -- filter: BlackList + +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -187,6 +188,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Observation Range Sanity Check - filter: Bounds Check filter variables: @@ -197,8 +199,9 @@ obs post filters: # maxvalue: 100.00000 action: name: reject + # Topography Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -210,8 +213,9 @@ obs post filters: options: channels: *iasi_metop-b_channels sensor: iasi_metop-b + # Transmittance Top Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -222,6 +226,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels + # Cloud Detection Check - filter: Bounds Check filter variables: @@ -361,6 +366,7 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # NSST Retrieval Check - filter: Bounds Check filter variables: @@ -438,8 +444,9 @@ obs post filters: maxvalue: 1.0e-12 action: name: reject + # Surface Jacobians Check -- filter: BlackList +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *iasi_metop-b_channels @@ -453,6 +460,7 @@ obs post filters: obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] sensor: iasi_metop-b + # Gross check - filter: Background Check filter variables: @@ -537,6 +545,7 @@ obs post filters: error parameter vector: *iasi_metop-b_oberr action: name: reject + # Useflag Check - filter: Bounds Check filter variables: @@ -612,5 +621,4 @@ obs post filters: minvalue: 1.0e-12 action: name: reject -passedBenchmark: 124731 - +passedBenchmark: 142616 # 124731 diff --git a/parm/atm/obs/testing/iasi_metop-b_noqc.yaml b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml new file mode 100644 index 000000000..700c46cd2 --- /dev/null +++ b/parm/atm/obs/testing/iasi_metop-b_noqc.yaml @@ -0,0 +1,158 @@ +obs space: + name: iasi_metop-b + obsdatain: + engine: + type: H5File + obsfile: iasi_metop-b_obs_${CDATE}_subset.nc4 + _source: gdas + obsdataout: + engine: + type: H5File + obsfile: iasi_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +geovals: + filename: iasi_metop-b_geoval_${CDATE}_subset.nc4 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: crtm/ +obs bias: + input file: iasi_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-b_tlap iasi_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *iasi_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +# covariance: +# minimal required obs number: 20 +# variance range: [1.0e-6, 10.0] +# step size: 1.0e-4 +# largest analysis variance: 10000.0 +# prior: +# inflation: +# ratio: 1.1 +# ratio for small dataset: 2.0 + +obs prior filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: &iasi_metop-b_oberr [ 1.38, 0.81, 0.75, 0.79, 0.72, 0.74, 0.68, 0.72, 0.65, 0.65, + 0.65, 0.69, 0.64, 0.64, 0.65, 0.67, 0.62, 0.61, 0.62, 0.64, 0.59, 0.76, + 1.22, 0.78, 0.64, 0.62, 0.61, 0.69, 0.65, 0.59, 0.61, 0.59, 0.68, 0.62, + 0.68, 4.38, 3.05, 2.31, 1.56, 1.33, 1.58, 0.93, 1.67, 0.72, 0.57, 0.58, + 0.55, 0.68, 0.59, 0.68, 0.59, 0.65, 0.58, 0.62, 0.64, 0.58, 0.64, 0.55, + 0.64, 0.5, 0.82, 0.59, 0.62, 0.51, 0.64, 0.52, 0.51, 0.51, 0.76, 0.52, + 0.57, 0.55, 0.69, 0.58, 0.65, 0.61, 0.59, 0.64, 0.76, 0.72, 1.05, 0.75, + 0.51, 0.65, 1.3, 0.69, 0.93, 1.49, 1.12, 0.68, 0.66, 0.67, 0.59, 0.59, + 0.69, 0.67, 0.64, 0.62, 0.72, 0.69, 0.66, 0.79, 0.78, 0.74, 0.88, 0.77, + 0.88, 0.86, 1, 0.87, 0.85, 0.88, 0.84, 0.84, 0.84, 0.8, 0.8, 0.87, + 0.98, 0.52, 0.65, 0.69, 0.61, 0.6, 0.67, 0.79, 0.62, 0.66, 0.7, 0.65, + 0.62, 0.61, 0.62, 0.53, 0.6, 0.68, 0.95, 0.63, 0.97, 0.65, 0.98, 0.58, + 0.73, 0.65, 0.85, 0.99, 0.76, 0.85, 0.97, 0.77, 0.62, 0.63, 1.21, 1.41, + 1.55, 1.78, 1.35, 1.14, 1.69, 1.79, 1.46, 1.63, 1.94, 2.01, 1.24, 1.76, + 1.26, 1.47, 1.9, 1.66, 2.13, 1.49, 1.52, 1.55, 1.96, 2.31, 2.33, 2.32, + 2.31, 2.33, 2.23, 2.33, 1.84, 2.29, 2.28, 2.28, 2.28, 2.26, 2.26, 2.26, + 2.27, 2.24, 2.23, 2.24, 2.26, 2.28, 2.28, 2.3, 2.15, 2.31, 2.37, 2.27, + 2.29, 2.29, 2.23, 2.28, 2.32, 2.32, 2.31, 2.32, 2.32, 2.31, 2.31, 2.28, + 2.29, 2.28, 2.26, 2.29, 2.27, 2.26, 2.25, 2.27, 2.24, 2.21, 2.24, 2.17, + 2.18, 2.17, 2.21, 1.99, 2.16, 2.2, 2.13, 2.12, 2.13, 2.1, 2.12, 2.11, + 2.09, 2.09, 2.08, 2.09, 2.04, 2.04, 2.1, 2.01, 2.05, 2.03, 2.06, 1.98, + 1.95, 1.94, 1.91, 1.7, 1.76, 1.77, 1.83, 2.04, 1.91, 1.99, 1.99, 2.07, + 2.02, 2.04, 2.1, 2.06, 2.18, 2.21, 2.24, 2.23, 2.23, 1.98, 2.2, 2.18, + 2.18, 2.21, 2.23, 2.24, 2.24, 2.25, 1.8, 2.24, 1.73, 1.73, 2.27, 1.67, + 2.21, 1.72, 2.23, 2.23, 2.23, 2.24, 2.23, 2.12, 2.17, 1.74, 2.02, 1.88, + 1.67, 1.73, 1.83, 1.82, 1.73, 1.83, 2.19, 1.84, 1.89, 1.6, 1.71, 1.86, + 1.85, 1.84, 1.87, 1.91, 1.52, 1.95, 1.87, 1.89, 1.91, 1.91, 1.93, 1.9, + 1.91, 1.9, 1.89, 1.89, 1.91, 1.9, 1.91, 1.91, 1.91, 1.93, 1.94, 1.91, + 1.92, 1.77, 1.91, 1.95, 1.19, 1.96, 1.98, 1.94, 1.55, 1.91, 1.92, 1.92, + 1.97, 1.93, 1.99, 1.86, 1.12, 1.93, 1.92, 1.95, 1.85, 1.84, 1.91, 1.12, + 1.82, 1.82, 1.95, 1.24, 1.94, 1.96, 1.21, 1.83, 1.96, 1.36, 1.96, 1.82, + 1.92, 1.68, 1.93, 1.23, 1.96, 1.93, 1.86, 1.41, 1.16, 1.6, 1.25, 1.2, + 1.65, 1.66, 1.87, 1.94, 1.96, 1.91, 1.25, 1.93, 1.91, 1.7, 0.99, 1.81, + 1.92, 1.95, 1.5, 1.47, 1.15, 1.58, 1.18, 1.82, 1.13, 1.83, 1.91, 1.26, + 1.27, 1.91, 1.45, 1.6, 1.29, 1.94, 1.94, 1.23, 1.95, 1.21, 1.94, 1.86, + 1.9, 1.33, 1.75, 2.02, 1.98, 2.03, 1.83, 1.5, 2.04, 2.02, 1.9, 2, 2.02, + 1.95, 1.93, 1.95, 1.95, 1.99, 2, 1.94, 1.96, 1.86, 1.92, 1.88, 1.86, + 1.84, 1.87, 1.77, 1.89, 1.89, 1.88, 1.94, 1.82, 1.79, 1.86, 2.06, 2.33, + 1.88, 1.86, 1.81, 1.8, 1.8, 1.86, 1.9, 2, 2.06, 2.1, 2.2, 2, 2.16, + 1.98, 1.8, 1.8, 1.85, 1.75, 2.04, 2.19, 2.14, 2.19, 1.86, 2.1, 2.11, + 2.18, 2.03, 2.28, 2.19, 2.26, 2.26, 2.21, 2.21, 2.26, 2.33, 2.27, 2.21, + 2.12, 2.23, 2.26, 2.25, 1.88, 2.26, 2.24, 2.36, 2.29, 2.35, 2.3, 2.27, + 2.08, 2.05, 2.27, 2.28, 2.27, 2.28, 1.97, 2.25, 2.25, 2.25, 2.31, 2.28, + 2.27, 2.13, 2.24, 2.28, 2.28, 2.41, 2.34, 9.32, 2.28, 2.38, 2.27, 2.27, + 2.39, 2.11, 2.09, 2.1, 2.06, 2.12, 2.08, 2, 1.93, 2.02, 2.55, 1.54, + 1.64, 1.51, 1.55, 2.82, 2.92, 2.55, 2.37, 1.85, 1.6, 1.72, 1.74, 1.79, + 1.9, 1.94, 2, 2.04, 2.08, 2.12, 2.13, 2.16, 2.18, 2.18, 2.2, 2.2, 2.41, + 2.39, 2.38, 2.4, 2.42, 2.41, 2.43, 2.45, 2.43, 2.45, 2.43, 2.4, 2.44, + 2.4, 2.42, 2.43, 2.45, 2.45, 2.45, 2.46, 2.45, 2.45, 2.43, 2.51, 2.48, + 2.48, 2.53, 2.46, 2.49, 2.5, 2.5, 2.5, 2.52, 2.52, 2.54, 2.5, 2.48, + 2.5, 2.55, 2.5, 2.48, 2.5, 2.5, 2.52, 2.52, 2.48, 2.5, 2.5, 2.52, 2.46, + 2.53, 9 ] +passedBenchmark: 790328 diff --git a/parm/ioda/bufr2ioda/atms_beamwidth.txt b/parm/ioda/bufr2ioda/atms_beamwidth.txt new file mode 100644 index 000000000..21a002d0c --- /dev/null +++ b/parm/ioda/bufr2ioda/atms_beamwidth.txt @@ -0,0 +1,36 @@ +# LOC = data/preproc/atms_beamwidth.dat +# WMO Satellite ID +224 +# Version identifier +1 +# Sampling distance (deg) +1.11 +# Number of channels to modify +22 +# Channel, beam width, new width, cutoff, nxaverage, nyaverage, QC dist +# Note: to use FFT technique, set new width and cutoff as required and set +# nxaverage and nyaverage to 0. To use simple averaging set new width to 0.0 +# and use nxaverage and nyaverage (e.g. 3 3 for 3x3 averaging). +# QC Dist gives number of points around missing value to flag as missing. + 1 5.2 3.33 0.4 0 0 11 + 2 5.2 3.33 0.4 0 0 11 + 3 2.2 3.33 0.0 0 0 5 + 4 2.2 3.33 0.0 0 0 5 + 5 2.2 3.33 0.0 0 0 5 + 6 2.2 3.33 0.0 0 0 5 + 7 2.2 3.33 0.0 0 0 5 + 8 2.2 3.33 0.0 0 0 5 + 9 2.2 3.33 0.0 0 0 5 +10 2.2 3.33 0.0 0 0 5 +11 2.2 3.33 0.0 0 0 5 +12 2.2 3.33 0.0 0 0 5 +13 2.2 3.33 0.0 0 0 5 +14 2.2 3.33 0.0 0 0 5 +15 2.2 3.33 0.0 0 0 5 +16 2.2 3.33 0.0 0 0 5 +17 1.1 3.33 0.0 0 0 5 +18 1.1 3.33 0.0 0 0 5 +19 1.1 3.33 0.0 0 0 5 +20 1.1 3.33 0.0 0 0 5 +21 1.1 3.33 0.0 0 0 5 +22 1.1 3.33 0.0 0 0 5 diff --git a/parm/ioda/bufr_atms.yaml b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml similarity index 93% rename from parm/ioda/bufr_atms.yaml rename to parm/ioda/bufr2ioda/bufr2ioda_atms.yaml index 7e8738e2f..0f1d69fb7 100644 --- a/parm/ioda/bufr_atms.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml @@ -1,7 +1,7 @@ observations: - obs space: name: bufr - obsdatain: "$(BUFR_in)" + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.atms.tm00.bufr_d" exports: variables: @@ -31,7 +31,7 @@ observations: # However, UFO/CRTM requred this variable to be float fieldOfViewNumber: query: "*/FOVN" - type: float +# type: float heightOfStation: query: "*/HMSL" @@ -51,9 +51,9 @@ observations: sensorScanAngle: sensorScanAngle: fieldOfViewNumber: "*/FOVN" - heightOfStation: "*/HMSL" scanStart: -52.725 scanStep: 1.110 + sensor: atms sensorChannelNumber: query: "*/ATMSCH/CHNM" @@ -64,7 +64,7 @@ observations: remappedBrightnessTemperature: fieldOfViewNumber: "*/FOVN" sensorChannelNumber: "*/ATMSCH/CHNM" - brightnessTemperature: "*/ATMSCH/TMBR" + brightnessTemperature: "*/ATMSCH/TMANT" obsTime: year: "*/YEAR" month: "*/MNTH" @@ -80,10 +80,11 @@ observations: map: _224: npp _225: n20 +# _226: n21 ioda: backend: netcdf - obsdataout: "$(IODA_out)" + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.atms_$(splitvar).tm00.nc" dimensions: - name: Channel diff --git a/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json b/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json new file mode 100644 index 000000000..17c0421ed --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.json @@ -0,0 +1,34 @@ +{ + "data_format" : "bufr_d", + "data_type" : "gnssro", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC003010" ], + "data_description" : "Satellite radio occultation data", + "data_provider" : "U.S. NOAA", + "satellite_info" : [ + { "satellite_name": "MetOp-A", "satellite_full_name": "Meteorological Operational Satellite - A GRAS", "satellite_id": 4 }, + { "satellite_name": "MetOp-B", "satellite_full_name": "Meteorological Operational Satellite - B GRAS", "satellite_id": 3 }, + { "satellite_name": "MetOp-C", "satellite_full_name": "Meteorological Operational Satellite - C GRAS", "satellite_id": 5 }, + { "satellite_name": "TSX", "satellite_full_name": "TerraSAR-X IGOR", "satellite_id": 42 }, + { "satellite_name": "TDX", "satellite_full_name": "TanDEM-X IGOR", "satellite_id": 43 }, + { "satellite_name": "PAZ", "satellite_full_name": "PAZ IGOR", "satellite_id": 44 }, + { "satellite_name": "Sentinel-6A", "satellite_full_name": "Sentinel-6 Michael Freilich", "satellite_id": 66 }, + { "satellite_name": "GeoOptics", "satellite_full_name": "GEOOPTICS CICERO OP1", "satellite_id": 265 }, + { "satellite_name": "GeoOptics", "satellite_full_name": "GEOOPTICS CICERO OP2", "satellite_id": 266 }, + { "satellite_name": "PlanetIQ", "satellite_full_name": "PLANETIQ GNOMES-A", "satellite_id": 267 }, + { "satellite_name": "PlanetIQ", "satellite_full_name": "PLANETIQ GNOMES-B", "satellite_id": 268 }, + { "satellite_name": "Spire", "satellite_full_name": "SPIRE LEMUR 3U CUBESAT", "satellite_id": 269 }, + { "satellite_name": "GraceC", "satellite_full_name": "Gravity Recovery and Climate Experiment C (GRACE-FO)", "satellite_id": 803 }, + { "satellite_name": "GraceD", "satellite_full_name": "Gravity Recovery and Climate Experiment D (GRACE-FO)", "satellite_id": 804 }, + { "satellite_name": "COSMIC-2 E1", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E1", "satellite_id": 750 }, + { "satellite_name": "COSMIC-2 E2", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E2", "satellite_id": 751 }, + { "satellite_name": "COSMIC-2 E3", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E3", "satellite_id": 752 }, + { "satellite_name": "COSMIC-2 E4", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E4", "satellite_id": 753 }, + { "satellite_name": "COSMIC-2 E5", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E5", "satellite_id": 754 }, + { "satellite_name": "COSMIC-2 E6", "satellite_full_name": "Constellation Observing System for Meteorology, Ionosphere, and Climate-2 E6", "satellite_id": 755 }, + { "satellite_name": "KOMPSAT-5", "satellite_full_name": "Korean Multi-purpose Satellite 5", "satellite_id": 825 } + ] +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml new file mode 100755 index 000000000..c15df2b62 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml @@ -0,0 +1,238 @@ +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.mtiasi.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteId: + query: "*/SAID" + + sensorId: + query: "*/SIID[1]" + + scanLineNumber: + query: "*/SLNM" + + gqisFlagQual: + query: "*/QGFQ" + + fieldOfViewNumber: + query: "*/FOVN" + + sensorScanPosition: + sensorScanPosition: + fieldOfViewNumber: "*/FOVN" + sensor: iasi + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + stationElevation: + query: "*/SELV" + type: float + + sensorViewAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.330 + scanStep: 3.334 + scanStepAdjust: 0.625 + sensor: iasi + + fractionOfClearPixelsInFOV: + query: "*/IASIL1CS{1}/FCPH" +# type: float +# transforms: +# - scale: 0.01 + + sensorChannelNumber: + query: "*/IASICHN/CHNM" + + # ObsValue + spectralRadiance: + spectralRadiance: + sensorChannelNumber: "*/IASICHN/CHNM" + startChannel: "*/IASIL1CB/STCH" + endChannel: "*/IASIL1CB/ENCH" + scaleFactor: "*/IASIL1CB/CHSF" + scaledSpectralRadiance: "*/IASICHN/SCRA" + + splits: + satId: + category: + variable: satelliteId + map: + _3: metop-b + _4: metop-a + _5: metop-c + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/IASICHN" + + - name: Cluster + path: "*/IASIL1CS" + + - name: Band + path: "*/IASIL1CB" + + globals: + - name: "platformCommonName" + type: string + value: "Meteorological Operational Satellite" + + - name: "platformLongDescription" + type: string + value: "EUMETSAT Polar System in sunsynchronous orbit" + + - name: "source" + type: string + value: "MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP)" + + - name: "sourceFiles" + type: string + value: "gdas.t00z.mtiasi.tm00.bufr_d" + + - name: "datetimeReference" + type: string + value: "2021-08-01T00:00:00Z" + + - name: "sensor" + type: string + value: "IASI" + + - name: "processingLevel" + type: string + value: "Level-1C" + + - name: "converter" + type: string + value: "BUFR" + + variables: + + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [ -90, 90 ] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [ -180, 180 ] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteId + longName: "Satellite Identifier" + + - name: "MetaData/instrumentIdentifier" + source: variables/sensorId + longName: "Satellite Instrument Identifier" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/qualityFlags" + source: variables/gqisFlagQual + longName: "Individual IASI-System Quality Flag" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/sensorScanPosition" + source: variables/sensorScanPosition + longName: "Sensor Scan Position" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [ 0, 180 ] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [ 0, 90 ] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorViewAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/stationElevation" + source: variables/stationElevation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/fractionOfClearPixelsInFOV" + source: variables/fractionOfClearPixelsInFOV + longName: "Fraction of Clear Pixels in a Field of View" + units: "1" + range: [0, 100] + +# The unit from BUFR is W m-2 sr-1 m -- this is radiance per wavenumber +# - name: "ObsValue/spectralRadiance" + - name: "ObsValue/radiance" + source: variables/spectralRadiance + longName: "IASI Spectral Radiance" + units: "W m-2 sr-1" +# units: "W m-2 sr-1 m-1" + diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json new file mode 100644 index 000000000..964175368 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.json @@ -0,0 +1,15 @@ +{ + "data_format" : "bufr_d", + "data_type" : "satwnd", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005044", "NC005045", "NC005046" ], + "data_description" : "NC005044 JMA SATWIND, AHI HIM8 IR(LW)/VIS/WV-CT/WV-CS; NC005045 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS; NC005046 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS", + "data_provider" : "JMA", + "sensor_info" : { "sensor_name": "AHI", "sensor_full_name": "Advanced Himawari Imager", "sensor_id": 999 }, + "satellite_info" : [ + { "satellite_name": "H8", "satellite_full_name": "Himawari-8", "satellite_id": 173, "launch time": "YYYYMMDD" }, + ] +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json new file mode 100644 index 000000000..3fad3c258 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.json @@ -0,0 +1,16 @@ +{ + "data_format" : "bufr_d", + "data_type" : "satwnd", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005067", "NC005068", "NC005069" ], + "data_description" : "EUMETSAT SATWIND, SEVIRI IR(LW); EUMETSAT SATWIND, SEVIRI WV-IMG/DL; EUMETSAT SATWIND, SEVIRI VIS", + "data_provider" : "EUMETSAT", + "sensor_info" : { "sensor_name": "SEVIRI", "sensor_full_name": "Spinning Enhanced Visible and InfraRed Imager", "sensor_id": 999 }, + "satellite_info" : [ + { "satellite_name": "M8", "satellite_full_name": "METEOSAT-8", "satellite_id": 55, "launch time": "YYYYMMDD" }, + { "satellite_name": "M11", "satellite_full_name": "METEOSAT-11", "satellite_id": 70, "launch time": "YYYYMMDD" }, + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py b/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py new file mode 100755 index 000000000..1b256c4a8 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_gnssro_bufr.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 +# (C) Copyright 2023 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import sys +import os +import argparse +import json +import numpy as np +import numpy.ma as ma +import math +import calendar +import time +import datetime +from pyiodaconv import bufr +from collections import namedtuple +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# GPS-RO BUFR dump file +# ===================================================================== +# NC003010 | GPS-RO +# ==================================================================== + + +def Derive_stationIdentification(said, ptid): + + stid = [] + for i in range(len(said)): + newval = str(said[i]).zfill(4)+str(ptid[i]).zfill(4) + stid.append(str(newval)) + stid = np.array(stid).astype(dtype='str') + stid = ma.array(stid) + ma.set_fill_value(stid, "") + + return stid + + +def Compute_Grid_Location(degrees): + + for i in range(len(degrees)): + if degrees[i] <= 360 and degrees[i] >= -180: + degrees[i] = np.deg2rad(degrees[i]) + rad = degrees + + return rad + + +def Compute_imph(impp, elrc): + + imph = (impp - elrc).astype(np.float32) + + return imph + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # ========================================= + # Get parameters from configuration + # ========================================= + data_format = config["data_format"] + data_type = config["data_type"] + bufr_data_type = "gpsro" + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + satellite_info_array = config["satellite_info"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + bufrfile = f"{cycle_type}.t{hh}z.{bufr_data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), + 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.debug(f"Making QuerySet ...") + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/ROSEQ1/CLATH') + q.add('longitude', '*/ROSEQ1/CLONH') + q.add('gridLatitude', '*/ROSEQ1/CLATH') + q.add('gridLongitude', '*/ROSEQ1/CLONH') + q.add('year', '*/YEAR') + q.add('year2', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteIdentifier', '*/SAID') + q.add('satelliteInstrument', '*/SIID') + q.add('satelliteConstellationRO', '*/SCLF') + q.add('satelliteTransmitterId', '*/PTID') + q.add('earthRadiusCurvature', '*/ELRC') +# q.add('observationSequenceNum', '*/SEQNUM') + q.add('geoidUndulation', '*/GEODU') + q.add('height', '*/ROSEQ3/HEIT') + q.add('impactParameterRO_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/IMPP') + q.add('impactParameterRO_roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/IMPP') + q.add('impactParameterRO_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/IMPP') + q.add('frequency__roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/MEFR') + q.add('frequency__roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/MEFR') + q.add('frequency__roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/MEFR') +# q.add('pccf', '*/ROSEQ1/PCCF') + q.add('pccf', '*/PCCF[1]') + q.add('percentConfidence', '*/ROSEQ3/PCCF') + q.add('sensorAzimuthAngle', '*/BEARAZ') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + + # Quality Information + q.add('qualityFlags', '*/QFRO') + q.add('qfro', '*/QFRO') + q.add('satelliteAscendingFlag', '*/QFRO') + + # ObsValue + q.add('bendingAngle_roseq2repl1', '*/ROSEQ1/ROSEQ2{1}/BNDA[1]') + q.add('bendingAngle_roseq2repl2', '*/ROSEQ1/ROSEQ2{2}/BNDA[1]') + q.add('bendingAngle_roseq2repl3', '*/ROSEQ1/ROSEQ2{3}/BNDA[1]') + q.add('atmosphericRefractivity', '*/ROSEQ3/ARFR[1]') + + # ObsError + q.add('obsErrorBendingAngle1', '*/ROSEQ1/ROSEQ2{1}/BNDA[2]') + q.add('obsErrorBendingAngle2', '*/ROSEQ1/ROSEQ2{2}/BNDA[2]') + q.add('obsErrorBendingAngle3', '*/ROSEQ1/ROSEQ2{3}/BNDA[2]') + q.add('obsErrorAtmosphericRefractivity', '*/ROSEQ3/ARFR[2]') + + # ObsType + q.add('obsTypeBendingAngle', '*/SAID') + q.add('obsTypeAtmosphericRefractivity', '*/SAID') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for making QuerySet: {running_time} seconds") + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.debug(f"Executing QuerySet to get ResultSet ...") + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + logger.debug(f" ... Executing QuerySet: get MetaData: basic ...") + # MetaData + clath = r.get('latitude', 'latitude') + clonh = r.get('longitude', 'latitude') + gclath = r.get('gridLatitude', 'latitude') + gclonh = r.get('gridLongitude', 'latitude') + year = r.get('year', 'latitude') + year2 = r.get('year2') + mnth = r.get('month', 'latitude') + days = r.get('day', 'latitude') + hour = r.get('hour', 'latitude') + minu = r.get('minute', 'latitude') + seco = r.get('second', 'latitude') + said = r.get('satelliteIdentifier', 'latitude') + siid = r.get('satelliteInstrument', 'latitude') + sclf = r.get('satelliteConstellationRO', 'latitude') + ptid = r.get('satelliteTransmitterId', 'latitude') + elrc = r.get('earthRadiusCurvature', 'latitude') +# seqnum = r.get('observationSequenceNum') + geodu = r.get('geoidUndulation', 'latitude') + heit = r.get('height', 'height', type='float32').astype(np.float32) + impp1 = r.get('impactParameterRO_roseq2repl1', 'latitude') + impp2 = r.get('impactParameterRO_roseq2repl2', 'latitude') + impp3 = r.get('impactParameterRO_roseq2repl3', 'latitude') + mefr1 = r.get('frequency__roseq2repl1', 'latitude', + type='float32').astype(np.float32) + mefr2 = r.get('frequency__roseq2repl2', 'latitude', + type='float32').astype(np.float32) + mefr3 = r.get('frequency__roseq2repl3', 'latitude', + type='float32').astype(np.float32) + pccf = r.get('pccf', 'latitude', type='float32').astype(np.float32) + ref_pccf = r.get('percentConfidence', 'height') + bearaz = r.get('sensorAzimuthAngle', 'latitude') + + logger.debug(f" ... Executing QuerySet: get MetaData: processing center...") + # Processing Center + ogce = r.get('dataProviderOrigin', 'latitude') + + logger.debug(f" ... Executing QuerySet: get metadata: data quality \ + information ...") + # Quality Information + qfro = r.get('qualityFlags', 'latitude') + qfro2 = r.get('qualityFlags', 'latitude', type='float32').astype(np.float32) + satasc = r.get('satelliteAscendingFlag', 'latitude') + + logger.debug(f" ... Executing QuerySet: get ObsValue: Bending Angle ...") + # ObsValue + # Bending Angle + bnda1 = r.get('bendingAngle_roseq2repl1', 'latitude') + bnda2 = r.get('bendingAngle_roseq2repl2', 'latitude') + bnda3 = r.get('bendingAngle_roseq2repl3', 'latitude') + arfr = r.get('atmosphericRefractivity', 'height') + + # ObsError + # Bending Angle + bndaoe1 = r.get('obsErrorBendingAngle1', 'latitude') + bndaoe2 = r.get('obsErrorBendingAngle2', 'latitude') + bndaoe3 = r.get('obsErrorBendingAngle3', 'latitude') + arfroe = r.get('obsErrorAtmosphericRefractivity', 'height') + + # ObsType + # Bending Angle + bndaot = r.get('obsTypeBendingAngle', 'latitude') + arfrot = r.get('obsTypeBendingAngle', 'latitude') + + logger.debug(f" ... Executing QuerySet: get datatime: observation time ...") + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', + 'second', 'latitude').astype(np.int64) + + logger.debug(f" ... Executing QuerySet: Done!") + + logger.debug(f" ... Executing QuerySet: Check BUFR variable generic \ + dimension and type ...") + # Check BUFR variable generic dimension and type + logger.debug(f" clath shape, type = {clath.shape}, {clath.dtype}") + logger.debug(f" clonh shape, type = {clonh.shape}, {clonh.dtype}") + logger.debug(f" gclath shape, type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh shape, type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" year shape, type = {year.shape}, {year.dtype}") + logger.debug(f" mnth shape, type = {mnth.shape}, {mnth.dtype}") + logger.debug(f" days shape, type = {days.shape}, {days.dtype}") + logger.debug(f" hour shape, type = {hour.shape}, {hour.dtype}") + logger.debug(f" minu shape, type = {minu.shape}, {minu.dtype}") + logger.debug(f" seco shape, type = {seco.shape}, {seco.dtype}") + logger.debug(f" said shape, type = {said.shape}, {said.dtype}") + logger.debug(f" siid shape, type = {siid.shape}, {siid.dtype}") + logger.debug(f" sclf shape, type = {sclf.shape}, {sclf.dtype}") + logger.debug(f" ptid shape, type = {ptid.shape}, {ptid.dtype}") + logger.debug(f" elrc shape, type = {elrc.shape}, {elrc.dtype}") + logger.debug(f" geodu shape, type = {geodu.shape}, {geodu.dtype}") + logger.debug(f" heit shape, type = {heit.shape}, {heit.dtype}") + logger.debug(f" impp1 shape, type = {impp1.shape}, {impp1.dtype}") + logger.debug(f" impp3 shape, type = {impp3.shape}, {impp3.dtype}") + logger.debug(f" mefr1 shape, type = {mefr1.shape}, {mefr1.dtype}") + logger.debug(f" mefr3 shape, type = {mefr3.shape}, {mefr3.dtype}") + logger.debug(f" pccf shape, type = {pccf.shape}, {pccf.dtype}") + logger.debug(f" pccf shape, fill = {pccf.fill_value}") + logger.debug(f" ref_pccf shape, type = {ref_pccf.shape}, \ + {ref_pccf.dtype}") + logger.debug(f" bearaz shape, type = {bearaz.shape}, {bearaz.dtype}") + + logger.debug(f" ogce shape, type = {ogce.shape}, {ogce.dtype}") + + logger.debug(f" qfro shape, type = {qfro.shape}, {qfro.dtype}") + logger.debug(f" satasc shape, type = {satasc.shape}, {satasc.dtype}") + + logger.debug(f" bnda1 shape, type = {bnda1.shape}, {bnda1.dtype}") + logger.debug(f" bnda3 shape, type = {bnda3.shape}, {bnda3.dtype}") + logger.debug(f" arfr shape, type = {arfr.shape}, {arfr.dtype}") + + logger.debug(f" bndaoe1 shape, type = {bndaoe1.shape}, \ + {bndaoe1.dtype}") + logger.debug(f" bndaoe3 shape, type = {bndaoe3.shape}, \ + {bndaoe3.dtype}") + logger.debug(f" arfroe shape, type = {arfr.shape}, {arfr.dtype}") + + logger.debug(f" bndaot shape, type = {bndaot.shape}, {bndaot.dtype}") + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for executing QuerySet to get ResultSet: \ + {running_time} seconds") + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.debug(f"Creating derived variables - stationIdentification") + stid = Derive_stationIdentification(said, ptid) + + logger.debug(f" stid shape,type = {stid.shape}, {stid.dtype}") + + logger.debug(f"Creating derived variables - Grid Latitude / Longitude ...") + gclonh = Compute_Grid_Location(gclonh) + gclath = Compute_Grid_Location(gclath) + + logger.debug(f" gclonh shape,type = {gclonh.shape}, {gclonh.dtype}") + logger.debug(f" gclath shape,type = {gclath.shape}, {gclath.dtype}") + logger.debug(f" gclonh min/max = {gclonh.min()}, {gclonh.max()}") + logger.debug(f" gclath min/max = {gclath.min()}, {gclath.max()}") + + logger.debug(f"Creating derived variables - imph ...") + imph1 = Compute_imph(impp1, elrc) + imph2 = Compute_imph(impp2, elrc) + imph3 = Compute_imph(impp3, elrc) + + logger.debug(f" imph1 shape,type = {imph1.shape}, {imph1.dtype}") + logger.debug(f" imph3 shape,type = {imph3.shape}, {imph3.dtype}") + logger.debug(f" imph1 min/max = {imph1.min()}, {imph1.max()}") + logger.debug(f" imph3 min/max = {imph3.min()}, {imph3.max()}") + + logger.debug(f"Keep bending angle with Freq = 0.0") + for i in range(len(said)): + if (mefr2[i] == 0.0): + bnda1[i] = bnda2[i] + mefr1[i] = mefr2[i] + impp1[i] = impp2[i] + imph1[i] = imph2[i] + bndaoe1[i] = bndaoe2[i] + if (mefr3[i] == 0.0): + bnda1[i] = bnda3[i] + mefr1[i] = mefr3[i] + impp1[i] = impp3[i] + imph1[i] = imph3[i] + bndaoe1[i] = bndaoe3[i] + + logger.debug(f" new bnda1 shape, type, min/max {bnda1.shape}, \ + {bnda1.dtype}, {bnda1.min()}, {bnda1.max()}") + logger.debug(f" new mefr1 shape, type, min/max {mefr1.shape}, \ + {mefr1.dtype}, {mefr1.min()}, {mefr1.max()}") + logger.debug(f" new impp1 shape, type, min/max {impp1.shape}, \ + {impp1.dtype}, {impp1.min()}, {impp1.max()}") + logger.debug(f" new imph1 shape, type, min/max {imph1.shape}, \ + {imph1.dtype}, {imph1.min()}, {imph1.max()}") + logger.debug(f" new bndaoe1 shape, type, min/max {bndaoe1.shape}, \ + {bndaoe1.dtype}, {bndaoe1.min()}, {bndaoe1.max()}") + +# find ibit for qfro (16bit from left to right) + bit3 = [] + bit5 = [] + bit6 = [] + for quality in qfro: + if quality & 8192 > 0: + bit3.append(1) + else: + bit3.append(0) + + if quality & 2048 > 0: + bit5.append(1) + else: + bit5.append(0) + + if quality & 1024 > 0: + bit6.append(1) + else: + bit6.append(0) + bit3 = np.array(bit3) + bit5 = np.array(bit5) + bit6 = np.array(bit6) + logger.debug(f" new bit3 shape, type, min/max {bit3.shape}, \ + {bit3.dtype}, {bit3.min()}, {bit3.max()}") + +# overwrite satelliteAscendingFlag and QFRO + for quality in range(len(bit3)): + satasc[quality] = 0 + qfro2[quality] = 0.0 + if bit3[quality] == 1: + satasc[quality] = 1 + if (bit5[quality] == 1) or (bit6[quality] == 1): + qfro2[quality] = 1.0 + + logger.debug(f" new satasc shape, type, min/max {satasc.shape}, \ + {satasc.dtype}, {satasc.min()}, {satasc.max()}") + logger.debug(f" new qfro2 shape, type, min/max {qfro2.shape}, \ + {qfro2.dtype}, {qfro2.min()}, {qfro2.max()}, {qfro2.fill_value}") + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for creating derived variables: {running_time} \ + seconds") + + # ===================================== + # Create IODA ObsSpace + # Write IODA output + # ===================================== + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(said) + logger.debug(f" ... Number of Unique satellite identifiers: \ + {len(unique_satids)}") + logger.debug(f" ... Unique satellite identifiers: {unique_satids}") + + # Create the dimensions + dims = {'Location': np.arange(0, clath.shape[0])} + + iodafile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.debug(f" ... ... Create OUTPUT file: {OUTPUT_PATH}") + + path, fname = os.path.split(OUTPUT_PATH) + if path and not os.path.exists(path): + os.makedirs(path) + + # Create IODA ObsSpace + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug(f" ... ... Create global attributes") + obsspace.write_attr('data_format', data_format) + obsspace.write_attr('data_type', data_type) + obsspace.write_attr('subsets', subsets) + obsspace.write_attr('cycle_type', cycle_type) + obsspace.write_attr('cycle_datetime', cycle) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('converter', os.path.basename(__file__)) + + # Create IODA variables + logger.debug(f" ... ... Create variables: name, type, units, & attributes") + # Longitude + obsspace.create_var('MetaData/longitude', dtype=clonh.dtype, + fillval=clonh.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(clonh) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=clath.dtype, + fillval=clath.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(clath) + + # Grid Longitude + obsspace.create_var('MetaData/gridLongitude', dtype=gclonh.dtype, + fillval=gclonh.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-3.14159265, 3.14159265], + dtype=np.float32)) \ + .write_attr('long_name', 'Grid Longitude') \ + .write_data(gclonh) + + # Grid Latitude + obsspace.create_var('MetaData/gridLatitude', dtype=gclath.dtype, + fillval=gclath.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('valid_range', np.array([-1.570796325, 1.570796325], + dtype=np.float32)) \ + .write_attr('long_name', 'Grid Latitude') \ + .write_data(gclath) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, + fillval=timestamp.fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp) + + # Station Identification + obsspace.create_var('MetaData/stationIdentification', dtype=stid.dtype, + fillval=stid.fill_value) \ + .write_attr('long_name', 'Station Identification') \ + .write_data(stid) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=said.dtype, + fillval=said.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(said) + + # Satellite Instrument + obsspace.create_var('MetaData/satelliteInstrument', dtype=siid.dtype, + fillval=siid.fill_value) \ + .write_attr('long_name', 'Satellite Instrument') \ + .write_data(siid) + + # Satellite Constellation RO + obsspace.create_var('MetaData/satelliteConstellationRO', dtype=sclf.dtype, + fillval=sclf.fill_value) \ + .write_attr('long_name', 'Satellite Constellation RO') \ + .write_data(sclf) + + # Satellite Transmitter ID + obsspace.create_var('MetaData/satelliteTransmitterId', dtype=ptid.dtype, + fillval=ptid.fill_value) \ + .write_attr('long_name', 'Satellite Transmitter Id') \ + .write_data(ptid) + + # Earth Radius Curvature + obsspace.create_var('MetaData/earthRadiusCurvature', dtype=elrc.dtype, + fillval=elrc.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Earth Radius of Curvature') \ + .write_data(elrc) + + # Geoid Undulation + obsspace.create_var('MetaData/geoidUndulation', dtype=geodu.dtype, + fillval=geodu.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Geoid Undulation') \ + .write_data(geodu) + + # Height + obsspace.create_var('MetaData/height', dtype=heit.dtype, + fillval=heit.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height') \ + .write_data(heit) + + # Impact Parameter RO + obsspace.create_var('MetaData/impactParameterRO', dtype=impp1.dtype, + fillval=impp1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Parameter RO') \ + .write_data(impp1) + + # Impact Height RO + obsspace.create_var('MetaData/impactHeightRO', dtype=imph1.dtype, + fillval=imph1.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Impact Height RO') \ + .write_data(imph1) + + # Impact Height RO + obsspace.create_var('MetaData/frequency', dtype=mefr1.dtype, + fillval=mefr1.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Frequency') \ + .write_data(mefr1) + + # PCCF Percent Confidence + obsspace.create_var('MetaData/pccf', dtype=pccf.dtype, + fillval=pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Profile Percent Confidence') \ + .write_data(pccf) + + # PCCF Ref Percent Confidence + obsspace.create_var('MetaData/percentConfidence', dtype=ref_pccf.dtype, + fillval=ref_pccf.fill_value) \ + .write_attr('units', '%') \ + .write_attr('long_name', 'Ref Percent Confidence') \ + .write_data(ref_pccf) + + # Azimuth Angle + obsspace.create_var('MetaData/sensorAzimuthAngle', dtype=bearaz.dtype, + fillval=bearaz.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('long_name', 'Percent Confidence') \ + .write_data(bearaz) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce.dtype, + fillval=ogce.fill_value) \ + .write_attr('long_name', 'Identification of Originating Center') \ + .write_data(ogce) + + # Quality: Quality Flags + obsspace.create_var('MetaData/qfro', dtype=qfro.dtype, + fillval=qfro.fill_value) \ + .write_attr('long_name', 'QFRO') \ + .write_data(qfro) + + obsspace.create_var('MetaData/qualityFlags', dtype=qfro2.dtype, + fillval=qfro2.fill_value) \ + .write_attr('long_name', 'Quality Flags for QFRO bit5 and bit6') \ + .write_data(qfro2) + + # Quality: Satellite Ascending Flag + obsspace.create_var('MetaData/satelliteAscendingFlag', dtype=satasc.dtype, + fillval=satasc.fill_value) \ + .write_attr('long_name', 'Satellite Ascending Flag') \ + .write_data(satasc) + + # ObsValue: Bending Angle + obsspace.create_var('ObsValue/bendingAngle', dtype=bnda1.dtype, + fillval=bnda1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle') \ + .write_data(bnda1) + + # ObsValue: Atmospheric Refractivity + obsspace.create_var('ObsValue/atmosphericRefractivity', dtype=arfr.dtype, + fillval=arfr.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity') \ + .write_data(arfr) + + # ObsError: Bending Angle + obsspace.create_var('ObsError/bendingAngle', dtype=bndaoe1.dtype, + fillval=bndaoe1.fill_value) \ + .write_attr('units', 'radians') \ + .write_attr('long_name', 'Bending Angle Obs Error') \ + .write_data(bndaoe1) + + # ObsError: Atmospheric Refractivity + obsspace.create_var('ObsError/atmosphericRefractivity', dtype=arfroe.dtype, + fillval=arfroe.fill_value) \ + .write_attr('units', 'N-units') \ + .write_attr('long_name', 'Atmospheric Refractivity ObsError') \ + .write_data(arfroe) + + # ObsType: Bending Angle + obsspace.create_var('ObsType/BendingAngle', dtype=bndaot.dtype, + fillval=bndaot.fill_value) \ + .write_attr('long_name', 'Bending Angle ObsType') \ + .write_data(bndaot) + + # ObsType: Atmospheric Refractivity + obsspace.create_var('ObsType/atmosphericRefractivity', dtype=arfrot.dtype, + fillval=arfrot.fill_value) \ + .write_attr('long_name', 'Atmospheric Refractivity ObsType') \ + .write_data(arfrot) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Running time for splitting and output IODA for gnssro bufr: \ + {running_time} seconds") + + logger.debug("All Done!") + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, + help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', + help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('bufr2ioda_gnssro.py', level=log_level, + colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py new file mode 100755 index 000000000..8380b12aa --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_ahi.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +import argparse +import numpy as np +import numpy.ma as ma +from pyiodaconv import bufr +import calendar +import json +import time +import math +import datetime +import os +from datetime import datetime +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# Satellite Winds (AMV) BUFR dump file for AHI/Himawari +# ==================================================================== +# All subsets contain all spectral bands: NC005044 NC005045 NC005046 +# ==================================================================== +# Spectral Band | Code (002023) | ObsType +# -------------------------------------------------------------------- +# IRLW (Freq < 5E+13) | Method 1 | 252 +# VIS | Method 2 | 242 +# WV Cloud Top | Method 3 | 250 +# WV Clear Sky/ Deep Layer | Method 5 | 250 +# ==================================================================== + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 250, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 250, obstype) # WVCT + obstype = np.where(swcm == 2, 242, obstype) # VIS + obstype = np.where(swcm == 1, 252, obstype) # IRLW + + if not np.any(np.isin(obstype, [242, 250, 252])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = 'BUFR to IODA Converter' + process_level = 'Level-2' + platform_description = 'Himawari-8' + sensor_description = 'Advanced Himawari Imager' + + logger.info(f'sensor_name = {sensor_name}') + logger.info(f'sensor_full_name = {sensor_full_name}') + logger.info(f'sensor_id = {sensor_id}') + logger.info(f'reference_time = {reference_time}') + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info('Making QuerySet') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/CLAT') + q.add('longitude', '*/CLON') + q.add('satelliteId', '*/SAID') + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteZenithAngle', '*/SAZA') + q.add('sensorCentralFrequency', '*/SCCF') + q.add('pressure', '*/PRLC') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + q.add('windGeneratingApplication', '*/QCPRMS[1]/GNAP') + +# # Quality Infomation (Quality Indicator w/o forecast) + q.add('qualityInformationWithoutForecast', '*/QCPRMS[1]/PCCF') + + # Wind Retrieval Method Information + q.add('windComputationMethod', '*/SWCM') + + # ObsValue + q.add('windDirection', '*/WDIR') + q.add('windSpeed', '*/WSPD') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f'Processing time for making QuerySet : {running_time} seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info('Executing QuerySet to get ResultSet') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + satid = r.get('satelliteId') + year = r.get('year') + month = r.get('month') + day = r.get('day') + hour = r.get('hour') + minute = r.get('minute') + second = r.get('second') + lat = r.get('latitude') + lon = r.get('longitude') + satzenang = r.get('satelliteZenithAngle') + pressure = r.get('pressure', type='float') + chanfreq = r.get('sensorCentralFrequency', type='float') + + # Processing Center + ogce = r.get('dataProviderOrigin') + ga = r.get('windGeneratingApplication') + + # Quality Information + qi = r.get('qualityInformationWithoutForecast', type='float') + # For AHI/Himawari data, qi w/o forecast (qifn) is packaged in same + # vector where ga == 102. Must conduct a search and extract the + # correct vector for gnap and qi + # 1. Find dimension-sizes of ga and qi (should be the same!) + gDim1, gDim2 = np.shape(ga) + qDim1, qDim2 = np.shape(qi) + logger.info(f'Generating Application and Quality Information SEARCH:') + logger.info(f'Dimension size of GNAP ({gDim1},{gDim2})') + logger.info(f'Dimension size of PCCF ({qDim1},{qDim2})') + # 2. Initialize gnap and qifn as None, and search for dimension of + # ga with values of 102. If the same column exists for qi, assign + # gnap to ga[:,i] and qifn to qi[:,i], else raise warning that no + # appropriate GNAP/PCCF combination was found + gnap = None + qifn = None + for i in range(gDim2): + if np.unique(ga[:, i].squeeze()) == 102: + if i <= qDim2: + logger.info(f'GNAP/PCCF found for column {i}') + gnap = ga[:, i].squeeze() + qifn = qi[:, i].squeeze() + else: + logger.info(f'ERROR: GNAP column {i} outside of PCCF dimension {qDim2}') + if (gnap is None) & (qifn is None): + logger.info(f'ERROR: GNAP == 102 NOT FOUND OR OUT OF PCCF DIMENSION-RANGE, WILL FAIL!') + + # Wind Retrieval Method Information + swcm = r.get('windComputationMethod') + + # ObsValue + # Wind direction and Speed + wdir = r.get('windDirection', type='float') + wspd = r.get('windSpeed') + + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second').astype(np.int64) + + # Check BUFR variable generic dimension and type + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for executing QuerySet to get ResultSet : {running_time} seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.info('Creating derived variables') + logger.debug('Creating derived variables - wind components (uob and vob)') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + logger.debug(f' uob min/max = {uob.min()} {uob.max()}') + logger.debug(f' vob min/max = {vob.min()} {vob.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + height = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + stnelev = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for creating derived variables : {running_time} seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info('Create IODA ObsSpace and Write IODA output based on satellite ID') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f'Number of Unique satellite identifiers: {len(unique_satids)}') + logger.info(f'Unique satellite identifiers: {unique_satids}') + + logger.debug(f'Loop through unique satellite identifier {unique_satids}') + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if (satellite_info["satellite_id"] == sat): + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower()+'_'+satellite_name.lower() + logger.debug(f'Split data for {satinst} satid = {sat}') + + if matched: + + # Define a boolean mask to subset data from the original data object + mask = satid == sat + # MetaData + lon2 = lon[mask] + lat2 = lat[mask] + timestamp2 = timestamp[mask] + satid2 = satid[mask] + satzenang2 = satzenang[mask] + chanfreq2 = chanfreq[mask] + obstype2 = obstype[mask] + pressure2 = pressure[mask] + height2 = height[mask] + stnelev2 = stnelev[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qifn2 = qifn[mask] + + # Method + swcm2 = swcm[mask] + + # ObsValue + wdir2 = wdir[mask] + wspd2 = wspd[mask] + uob2 = uob[mask] + vob2 = vob[mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f'Processing output for satid {sat}') + + # Create the dimensions + dims = { + 'Location': np.arange(0, wdir2.shape[0]) + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f'Create output file : {OUTPUT_PATH}') + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug('Write global attributes') + obsspace.write_attr('Converter', converter) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(timestamp2_min), str(timestamp2_max)]) + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('platformCommonName', satellite_name) + obsspace.write_attr('sensorCommonName', sensor_name) + obsspace.write_attr('processingLevel', process_level) + obsspace.write_attr('platformLongDescription', platform_description) + obsspace.write_attr('sensorLongDescription', sensor_description) + + # Create IODA variables + logger.debug('Write variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon2.dtype, fillval=lon2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat.dtype, fillval=lat2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=int64_fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=satid2.dtype, fillval=satid2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(satid2) + + # Sensor Zenith Angle + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Satellite Zenith Angle') \ + .write_data(satzenang2) + + # Sensor Centrall Frequency + obsspace.create_var('MetaData/sensorCentralFrequency', dtype=chanfreq2.dtype, fillval=chanfreq2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Satellite Channel Center Frequency') \ + .write_data(chanfreq2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Percent Confidence - Quality Information Without Forecast + obsspace.create_var('MetaData/qualityInformationWithoutForecast', dtype=qifn2.dtype, fillval=qifn2.fill_value) \ + .write_attr('long_name', 'Quality Information Without Forecast') \ + .write_data(qifn2) + + # Wind Computation Method + obsspace.create_var('MetaData/windComputationMethod', dtype=swcm2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Satellite-derived Wind Computation Method') \ + .write_data(swcm2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windNorthward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure2.dtype, fillval=pressure2.fill_value) \ + .write_attr('units', 'pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure2) + + # Height (mimic prepbufr) + obsspace.create_var('MetaData/height', dtype=height2.dtype, fillval=height2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height of Observation') \ + .write_data(height2) + + # Station Elevation (mimic prepbufr) + obsspace.create_var('MetaData/stationElevation', dtype=stnelev2.dtype, fillval=stnelev2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelev2) + + # U-Wind Component + obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind Component') \ + .write_data(uob2) + + # V-Wind Component + obsspace.create_var('ObsValue/windNorthward', dtype=vob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind Component') \ + .write_data(vob2) + + end_time = time.time() + running_time = end_time - start_time + total_ob_processed += len(satid2) + logger.debug(f'Number of observation processed : {len(satid2)}') + logger.debug(f'Processing time for splitting and output IODA for {satinst} : {running_time} seconds') + + else: + logger.info(f"Do not find this satellite id in the configuration: satid = {sat}") + + logger.info("All Done!") + logger.info(f'Total number of observation processed : {total_ob_processed}') + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_satwind_amv_ahi.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py index 88f0d7a5b..7e1207f54 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_goes.py @@ -124,7 +124,7 @@ def bufr_to_ioda(config, logger): q.add('hour', '*/HOUR') q.add('minute', '*/MINU') q.add('second', '*/SECO') - q.add('sensorZenithAngle', '*/SAZA') + q.add('satelliteZenithAngle', '*/SAZA') q.add('sensorCentralFrequency', '*/SCCF') q.add('pressure', '*/PRLC[1]') @@ -173,7 +173,7 @@ def bufr_to_ioda(config, logger): second = r.get('second') lat = r.get('latitude') lon = r.get('longitude') - satzenang = r.get('sensorZenithAngle') + satzenang = r.get('satelliteZenithAngle') pressure = r.get('pressure', type='float') chanfreq = r.get('sensorCentralFrequency', type='float') @@ -357,10 +357,10 @@ def bufr_to_ioda(config, logger): .write_data(satid2) # Sensor Zenith Angle - obsspace.create_var('MetaData/sensorZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ .write_attr('units', 'degree') \ .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ - .write_attr('long_name', 'Sensor Zenith Angle') \ + .write_attr('long_name', 'Satellite Zenith Angle') \ .write_data(satzenang2) # Sensor Centrall Frequency @@ -428,18 +428,6 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Station Elevation') \ .write_data(stnelev2) - # Wind Speed - obsspace.create_var('ObsValue/windSpeed', dtype=wspd2.dtype, fillval=wspd2.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Wind Speed') \ - .write_data(wspd2) - - # Wind Direction - obsspace.create_var('ObsValue/windDirection', dtype=wdir2.dtype, fillval=wdir2.fill_value) \ - .write_attr('units', 'degrees') \ - .write_attr('long_name', 'Wind Direction') \ - .write_data(wdir2) - # U-Wind Component obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ .write_attr('units', 'm s-1') \ diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py new file mode 100755 index 000000000..c28274948 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_amv_seviri.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +import argparse +import numpy as np +import numpy.ma as ma +from pyiodaconv import bufr +import calendar +import json +import time +import math +import datetime +import os +from datetime import datetime +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +# ==================================================================== +# Satellite Winds (AMV) BUFR dump file for SEVIRI/METEOSAT +# ==================================================================== +# All subsets contain all spectral bands: NC005067 NC005068 NC005069 +# ==================================================================== +# Spectral Band | Code (002023) | ObsType +# -------------------------------------------------------------------- +# IRLW (Freq < 5E+13) | Method 1 | 253 +# VIS | Method 2 | 243 +# WV Cloud Top | Method 3 | 254 +# WV Clear Sky/ Deep Layer | Method 5 | 254 +# ==================================================================== + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + +def Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd): + + uob = (-wspd * np.sin(np.radians(wdir))).astype(np.float32) + vob = (-wspd * np.cos(np.radians(wdir))).astype(np.float32) + + return uob, vob + + +def Get_ObsType(swcm, chanfreq): + + obstype = swcm.copy() + + # Use numpy vectorized operations + obstype = np.where(swcm == 5, 254, obstype) # WVCA/DL + obstype = np.where(swcm == 3, 254, obstype) # WVCT + obstype = np.where(swcm == 2, 243, obstype) # VIS + obstype = np.where(swcm == 1, 253, obstype) # IRLW + + if not np.any(np.isin(obstype, [243, 253, 254])): + raise ValueError("Error: Unassigned ObsType found ... ") + + return obstype + + +def bufr_to_ioda(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = 'BUFR to IODA Converter' + process_level = 'Level-2' + platform_description = 'METEOSAT-8' + sensor_description = 'Spinning Enhanced Visible and InfraRed Imager' + + logger.info(f'sensor_name = {sensor_name}') + logger.info(f'sensor_full_name = {sensor_full_name}') + logger.info(f'sensor_id = {sensor_id}') + logger.info(f'reference_time = {reference_time}') + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + + # ============================================ + # Make the QuerySet for all the data we want + # ============================================ + start_time = time.time() + + logger.info('Making QuerySet') + q = bufr.QuerySet(subsets) + + # MetaData + q.add('latitude', '*/CLATH') + q.add('longitude', '*/CLONH') + q.add('satelliteId', '*/SAID') + q.add('year', '*/YEAR') + q.add('month', '*/MNTH') + q.add('day', '*/DAYS') + q.add('hour', '*/HOUR') + q.add('minute', '*/MINU') + q.add('second', '*/SECO') + q.add('satelliteZenithAngle', '*/SAZA') + q.add('sensorCentralFrequency', '*/SCCF') + q.add('pressure', '*/PRLC[1]') + + # Processing Center + q.add('dataProviderOrigin', '*/OGCE') + +# # Quality Infomation (Quality Indicator w/o forecast) + q.add('qualityInformationWithoutForecast', '*/AMVQIC{2}/PCCF') + + # Wind Retrieval Method Information + q.add('windComputationMethod', '*/SWCM') + + # ObsValue + q.add('windDirection', '*/WDIR') + q.add('windSpeed', '*/WSPD') + + end_time = time.time() + running_time = end_time - start_time + logger.debug(f'Processing time for making QuerySet : {running_time} seconds') + + # ============================================================== + # Open the BUFR file and execute the QuerySet to get ResultSet + # Use the ResultSet returned to get numpy arrays of the data + # ============================================================== + start_time = time.time() + + logger.info('Executing QuerySet to get ResultSet') + with bufr.File(DATA_PATH) as f: + r = f.execute(q) + + # MetaData + satid = r.get('satelliteId') + year = r.get('year') + month = r.get('month') + day = r.get('day') + hour = r.get('hour') + minute = r.get('minute') + second = r.get('second') + lat = r.get('latitude') + lon = r.get('longitude') + satzenang = r.get('satelliteZenithAngle') + pressure = r.get('pressure', type='float') + chanfreq = r.get('sensorCentralFrequency', type='float') + + # Processing Center + ogce = r.get('dataProviderOrigin') + + # Quality Information + qifn = r.get('qualityInformationWithoutForecast', type='float') + + # Wind Retrieval Method Information + swcm = r.get('windComputationMethod') + + # ObsValue + # Wind direction and Speed + wdir = r.get('windDirection', type='float') + wspd = r.get('windSpeed') + + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second').astype(np.int64) + + # Check BUFR variable generic dimension and type + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for executing QuerySet to get ResultSet : {running_time} seconds') + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.info('Creating derived variables') + logger.debug('Creating derived variables - wind components (uob and vob)') + + uob, vob = Compute_WindComponents_from_WindDirection_and_WindSpeed(wdir, wspd) + + logger.debug(f' uob min/max = {uob.min()} {uob.max()}') + logger.debug(f' vob min/max = {vob.min()} {vob.max()}') + + obstype = Get_ObsType(swcm, chanfreq) + + height = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + stnelev = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for creating derived variables : {running_time} seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info('Create IODA ObsSpace and Write IODA output based on satellite ID') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f'Number of Unique satellite identifiers: {len(unique_satids)}') + logger.info(f'Unique satellite identifiers: {unique_satids}') + + logger.debug(f'Loop through unique satellite identifier {unique_satids}') + total_ob_processed = 0 + for sat in unique_satids.tolist(): + start_time = time.time() + + matched = False + for satellite_info in satellite_info_array: + if (satellite_info["satellite_id"] == sat): + matched = True + satellite_id = satellite_info["satellite_id"] + satellite_name = satellite_info["satellite_name"] + satinst = sensor_name.lower()+'_'+satellite_name.lower() + logger.debug(f'Split data for {satinst} satid = {sat}') + + if matched: + + # Define a boolean mask to subset data from the original data object + mask = satid == sat + # MetaData + lon2 = lon[mask] + lat2 = lat[mask] + timestamp2 = timestamp[mask] + satid2 = satid[mask] + satzenang2 = satzenang[mask] + chanfreq2 = chanfreq[mask] + obstype2 = obstype[mask] + pressure2 = pressure[mask] + height2 = height[mask] + stnelev2 = stnelev[mask] + + # Processing Center + ogce2 = ogce[mask] + + # QC Info + qifn2 = qifn[mask] + + # Method + swcm2 = swcm[mask] + + # ObsValue + wdir2 = wdir[mask] + wspd2 = wspd[mask] + uob2 = uob[mask] + vob2 = vob[mask] + + # Timestamp Range + timestamp2_min = datetime.fromtimestamp(timestamp2.min()) + timestamp2_max = datetime.fromtimestamp(timestamp2.max()) + + # Check unique observation time + unique_timestamp2 = np.unique(timestamp2) + logger.debug(f'Processing output for satid {sat}') + + # Create the dimensions + dims = { + 'Location': np.arange(0, wdir2.shape[0]) + } + + # Create IODA ObsSpace + iodafile = f"{cycle_type}.t{hh}z.{data_type}.{satinst}.tm00.nc" + OUTPUT_PATH = os.path.join(ioda_dir, iodafile) + logger.info(f'Create output file : {OUTPUT_PATH}') + obsspace = ioda_ospace.ObsSpace(OUTPUT_PATH, mode='w', dim_dict=dims) + + # Create Global attributes + logger.debug('Write global attributes') + obsspace.write_attr('Converter', converter) + obsspace.write_attr('sourceFiles', bufrfile) + obsspace.write_attr('dataProviderOrigin', data_provider) + obsspace.write_attr('description', data_description) + obsspace.write_attr('datetimeReference', reference_time) + obsspace.write_attr('datetimeRange', [str(timestamp2_min), str(timestamp2_max)]) + obsspace.write_attr('sensor', sensor_id) + obsspace.write_attr('platform', satellite_id) + obsspace.write_attr('platformCommonName', satellite_name) + obsspace.write_attr('sensorCommonName', sensor_name) + obsspace.write_attr('processingLevel', process_level) + obsspace.write_attr('platformLongDescription', platform_description) + obsspace.write_attr('sensorLongDescription', sensor_description) + + # Create IODA variables + logger.debug('Write variables: name, type, units, and attributes') + # Longitude + obsspace.create_var('MetaData/longitude', dtype=lon2.dtype, fillval=lon2.fill_value) \ + .write_attr('units', 'degrees_east') \ + .write_attr('valid_range', np.array([-180, 180], dtype=np.float32)) \ + .write_attr('long_name', 'Longitude') \ + .write_data(lon2) + + # Latitude + obsspace.create_var('MetaData/latitude', dtype=lat2.dtype, fillval=lat2.fill_value) \ + .write_attr('units', 'degrees_north') \ + .write_attr('valid_range', np.array([-90, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Latitude') \ + .write_data(lat2) + + # Datetime + obsspace.create_var('MetaData/dateTime', dtype=np.int64, fillval=int64_fill_value) \ + .write_attr('units', 'seconds since 1970-01-01T00:00:00Z') \ + .write_attr('long_name', 'Datetime') \ + .write_data(timestamp2) + + # Satellite Identifier + obsspace.create_var('MetaData/satelliteIdentifier', dtype=satid2.dtype, fillval=satid2.fill_value) \ + .write_attr('long_name', 'Satellite Identifier') \ + .write_data(satid2) + + # Sensor Zenith Angle + obsspace.create_var('MetaData/satelliteZenithAngle', dtype=satzenang2.dtype, fillval=satzenang2.fill_value) \ + .write_attr('units', 'degree') \ + .write_attr('valid_range', np.array([0, 90], dtype=np.float32)) \ + .write_attr('long_name', 'Satellite Zenith Angle') \ + .write_data(satzenang2) + + # Sensor Centrall Frequency + obsspace.create_var('MetaData/sensorCentralFrequency', dtype=chanfreq2.dtype, fillval=chanfreq2.fill_value) \ + .write_attr('units', 'Hz') \ + .write_attr('long_name', 'Satellite Channel Center Frequency') \ + .write_data(chanfreq2) + + # Data Provider + obsspace.create_var('MetaData/dataProviderOrigin', dtype=ogce2.dtype, fillval=ogce2.fill_value) \ + .write_attr('long_name', 'Identification of Originating/Generating Center') \ + .write_data(ogce2) + + # Quality: Percent Confidence - Quality Information Without Forecast + obsspace.create_var('MetaData/qualityInformationWithoutForecast', dtype=qifn2.dtype, fillval=qifn2.fill_value) \ + .write_attr('long_name', 'Quality Information Without Forecast') \ + .write_data(qifn2) + + # Wind Computation Method + obsspace.create_var('MetaData/windComputationMethod', dtype=swcm2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Satellite-derived Wind Computation Method') \ + .write_data(swcm2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # ObsType based on computation method/spectral band + obsspace.create_var('ObsType/windNorthward', dtype=obstype2.dtype, fillval=swcm2.fill_value) \ + .write_attr('long_name', 'Observation Type based on Satellite-derived Wind Computation Method and Spectral Band') \ + .write_data(obstype2) + + # Pressure + obsspace.create_var('MetaData/pressure', dtype=pressure2.dtype, fillval=pressure2.fill_value) \ + .write_attr('units', 'pa') \ + .write_attr('long_name', 'Pressure') \ + .write_data(pressure2) + + # Height (mimic prepbufr) + obsspace.create_var('MetaData/height', dtype=height2.dtype, fillval=height2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Height of Observation') \ + .write_data(height2) + + # Station Elevation (mimic prepbufr) + obsspace.create_var('MetaData/stationElevation', dtype=stnelev2.dtype, fillval=stnelev2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelev2) + + # U-Wind Component + obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Eastward Wind Component') \ + .write_data(uob2) + + # V-Wind Component + obsspace.create_var('ObsValue/windNorthward', dtype=vob2.dtype, fillval=wspd2.fill_value) \ + .write_attr('units', 'm s-1') \ + .write_attr('long_name', 'Northward Wind Component') \ + .write_data(vob2) + + end_time = time.time() + running_time = end_time - start_time + total_ob_processed += len(satid2) + logger.debug(f'Number of observation processed : {len(satid2)}') + logger.debug(f'Processing time for splitting and output IODA for {satinst} : {running_time} seconds') + + else: + logger.info(f"Do not find this satellite id in the configuration: satid = {sat}") + + logger.info("All Done!") + logger.info(f'Total number of observation processed : {total_ob_processed}') + + +if __name__ == '__main__': + + start_time = time.time() + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger('BUFR2IODA_satwind_amv_ahi.py', level=log_level, colored_log=True) + + with open(args.config, "r") as json_file: + config = json.load(json_file) + + bufr_to_ioda(config, logger) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f"Total running time: {running_time} seconds") diff --git a/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py b/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py index 352dac6a8..abdeb6358 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_satwind_scat.py @@ -169,6 +169,9 @@ def bufr_to_ioda(config, logger): logger.debug('Creating derived variables - observation height') height = np.full_like(lat, 10.) + logger.debug('Creating derived variables - station elevation') + stnelv = np.full_like(lat, 0.) + end_time = time.time() running_time = end_time - start_time logger.debug(f'Processing time for creating derived variables : {running_time} seconds') @@ -209,6 +212,7 @@ def bufr_to_ioda(config, logger): timestamp2 = timestamp[mask] pressure2 = pressure[mask] height2 = height[mask] + stnelv2 = stnelv[mask] obstype2 = obstype[mask] # QC Info @@ -299,6 +303,12 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'Height of Observation') \ .write_data(height2) + # Station Elevation + obsspace.create_var('MetaData/stationElevation', dtype=stnelv2.dtype, fillval=stnelv2.fill_value) \ + .write_attr('units', 'm') \ + .write_attr('long_name', 'Station Elevation') \ + .write_data(stnelv2) + # ObsType based on sensor type obsspace.create_var('ObsType/windEastward', dtype=obstype2.dtype, fillval=obstype2.fill_value) \ .write_attr('long_name', 'PrepBUFR Report Type') \ @@ -309,18 +319,6 @@ def bufr_to_ioda(config, logger): .write_attr('long_name', 'PrepBUFR Report Type') \ .write_data(obstype2) - # Wind Speed - obsspace.create_var('ObsValue/windSpeed', dtype=wspd2.dtype, fillval=wspd2.fill_value) \ - .write_attr('units', 'm s-1') \ - .write_attr('long_name', 'Wind Speed at 10 Meters') \ - .write_data(wspd2) - - # Wind Direction - obsspace.create_var('ObsValue/windDirection', dtype=wdir2.dtype, fillval=wdir2.fill_value) \ - .write_attr('units', 'degrees') \ - .write_attr('long_name', 'Wind Direction at 10 Meters') \ - .write_data(wdir2) - # U-Wind Component obsspace.create_var('ObsValue/windEastward', dtype=uob2.dtype, fillval=uob2.fill_value) \ .write_attr('units', 'm s-1') \ diff --git a/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py b/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py index cd0dd1ce6..ce7b115ce 100755 --- a/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py +++ b/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py @@ -5,52 +5,30 @@ # and certain configuration parameters import argparse import os -from wxflow import Template, TemplateConstants, YAMLFile +from wxflow import Logger, parse_j2yaml, cast_strdict_as_dtypedict, save_as_yaml +from wxflow import Template, TemplateConstants +# initialize root logger +logger = Logger('gen_bufr2ioda_yaml.py', level='INFO', colored_log=True) -# list of satellite radiance BUFR files that need split by SatId -sat_list = [ - 'atms', - '1bamua', - '1bmhs', - 'crisf4', - 'iasidb', -] - -def gen_bufr_yaml(config): - # open the template input file - bufr_yaml = YAMLFile(path=config['template yaml']) - # determine if splits need in the output file path - obtype = config['obtype'] - if obtype in sat_list: - # split by satellite platform - obtype_out = f"{obtype}_{{splits/satId}}" - else: - obtype_out = obtype - # construct the output IODA file path - output_ioda = [ - config['run'], - f"t{config['cyc']:02}z", - obtype_out, - 'nc', - ] - output_ioda_str = '.'.join(output_ioda) - output_ioda_file = os.path.join(config['output dir'], output_ioda_str) - # construct the template substitution dict - substitutions = { - 'BUFR_in': config['input file'], - 'IODA_out': output_ioda_file, - } - # substitue templates - bufr_yaml = Template.substitute_structure(bufr_yaml, TemplateConstants.DOLLAR_PARENTHESES, substitutions.get) - # write out BUFR converter YAML file - bufr_yaml.save(config['output yaml file']) +def gen_bufr_yaml(config, template, output): + # read in templated YAML and do substitution + logger.info(f"Using {template} as input") + bufr_config = parse_j2yaml(template, config) + # need to do some special manipulation for the splits + substitutions = {'splitvar': '{splits/satId}'} + bufr_config = Template.substitute_structure(bufr_config, TemplateConstants.DOLLAR_PARENTHESES, substitutions.get) + save_as_yaml(bufr_config, output) + logger.info(f"Wrote to {output}") if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) + parser.add_argument('-t', '--template', type=str, help='Input YAML template', required=True) + parser.add_argument('-o', '--output', type=str, help='Output YAML file', required=True) args = parser.parse_args() - config = YAMLFile(path=args.config) - gen_bufr_yaml(config) + # get the config from your environment + config = cast_strdict_as_dtypedict(os.environ) + # call the parsing function + gen_bufr_yaml(config, args.template, args.output) diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 4d826359e..153cf2de5 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -1,14 +1,32 @@ #!/usr/bin/env python3 import argparse +import glob +import multiprocessing as mp import os +import shutil +from itertools import repeat from pathlib import Path from gen_bufr2ioda_json import gen_bufr_json +from gen_bufr2ioda_yaml import gen_bufr_yaml from wxflow import (Logger, Executable, cast_as_dtype, logit, to_datetime, datetime_to_YMDH, Task, rm_p) # Initialize root logger logger = Logger('run_bufr2ioda.py', level='INFO', colored_log=True) +# get parallel processing info +num_cores = mp.cpu_count() + + +def mp_bufr_converter(exename, configfile): + cmd = Executable(exename) + filetype = Path(configfile).suffix + if filetype == '.json': + cmd.add_default_arg('-c') + cmd.add_default_arg(configfile) + logger.info(f"Executing {cmd}") + cmd() + @logit(logger) def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): @@ -17,6 +35,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # Get gdasapp root directory DIR_ROOT = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) USH_IODA = os.path.join(DIR_ROOT, "ush", "ioda", "bufr2ioda") + BIN_GDAS = os.path.join(DIR_ROOT, "build", "bin") + DATA = os.getcwd() # Create output directory if it doesn't exist os.makedirs(COM_OBS, exist_ok=True) @@ -26,31 +46,66 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): 'RUN': RUN, 'current_cycle': current_cycle, 'DMPDIR': DMPDIR, - 'COM_OBS': COM_OBS + 'COM_OBS': COM_OBS, + 'PDY': current_cycle.strftime('%Y%m%d'), + 'cyc': current_cycle.strftime('%H'), } + # copy necessary fix files to runtime directory + shutil.copy(os.path.join(config_template_dir, "atms_beamwidth.txt"), + os.path.join(DATA, "atms_beamwidth.txt")) + # Specify observation types to be processed by a script - BUFR_py = ["satwind_amv_goes", "satwind_scat", "adpupa_prepbufr", "adpsfc_prepbufr", "sfcshp_prepbufr", "acft_profiles_prepbufr", - "gpsro_bufr", "conventional_prepbufr_ps", "bufr2ioda_subpfl_argo_profiles.py"] + BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) + BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] + BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] + config_files = [] + exename = [] for obtype in BUFR_py: logger.info(f"Convert {obtype}...") - json_output_file = os.path.join(COM_OBS, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + json_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") filename = 'bufr2ioda_' + obtype + '.json' template = os.path.join(config_template_dir, filename) gen_bufr_json(config, template, json_output_file) # Use the converter script for the ob type bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" - cmd = Executable(bufr2iodapy) - cmd.add_default_arg('-c') - cmd.add_default_arg(json_output_file) - logger.info(f"Executing {cmd}") - cmd() + + # append the values to the lists + config_files.append(json_output_file) + exename.append(bufr2iodapy) + + # Check if the converter was successful + # if os.path.exists(json_output_file): + # rm_p(json_output_file) + + # Specify observation types to be processed by the bufr2ioda executable + BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) + BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] + BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] + + for obtype in BUFR_yaml: + logger.info(f"Convert {obtype}...") + yaml_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.yaml") + filename = 'bufr2ioda_' + obtype + '.yaml' + template = os.path.join(config_template_dir, filename) + gen_bufr_yaml(config, template, yaml_output_file) + + # use the bufr2ioda executable for the ob type + bufr2iodaexe = BIN_GDAS + '/bufr2ioda.x' + + # append the values to the lists + config_files.append(yaml_output_file) + exename.append(bufr2iodaexe) # Check if the converter was successful - if os.path.exists(json_output_file): - rm_p(json_output_file) + # if os.path.exists(yaml_output_file): + # rm_p(yaml_output_file) + + # run everything in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_converter, zip(exename, config_files)) if __name__ == "__main__": From 99c431aa42022e91f12ff082d646fcb34f6a83cd Mon Sep 17 00:00:00 2001 From: emilyhcliu <36091766+emilyhcliu@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:21:08 -0500 Subject: [PATCH 22/22] Update observation YAMLs for conventional surface pressure (#803) This PR includes the following updates for conventional surface pressure 1. Add observation YAML under configuration 2. Update observation YAML under testing 3. Add conv_ps to 3dvar obs list 4. Set window shift to true (time boundary is inclusive) 5. Validate 3dvar for surface (station) pressure (end-to-end) - [QC validation](https://github.com/NOAA-EMC/GDASApp/pull/803#issuecomment-1851387894) - [Increments](https://github.com/NOAA-EMC/GDASApp/pull/803#issuecomment-1852496937) See [PR #792 ](https://github.com/NOAA-EMC/GDASApp/pull/792) for the BUFR converter work to convert the following surface pressure observation into IODA: - SFC - 181, 187 - SFCSHIP - 180, 183 - SONDE - 120 The related UFO evaluation (using GSI obs and geovals) results for surface pressure are documented [here](https://docs.google.com/presentation/d/1OvVMYOutxVyk0GE93Jzoh3hQVoj-dCgrwsVWDDlPT1s/edit?usp=sharing). --- parm/atm/obs/config/conv_ps.yaml | 142 +++++++++++++++------- parm/atm/obs/lists/gdas_prototype_3d.yaml | 1 + parm/atm/obs/testing/conv_ps.yaml | 62 ++++------ parm/atm/variational/3dvar_dripcg.yaml | 1 + 4 files changed, 122 insertions(+), 84 deletions(-) diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml index 8fa4ab185..ffae20073 100644 --- a/parm/atm/obs/config/conv_ps.yaml +++ b/parm/atm/obs/config/conv_ps.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)conventional_ps.prepbufr.nc obsdataout: engine: type: H5File - obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 + obsfile: $(DATA)/diags/diag_conventional_ps_prepbufr_{{ current_cycle | to_YMDH }}.nc io pool: max pool size: 1 simulated variables: [stationPressure] @@ -21,6 +21,11 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height +linear obs operator: + name: Identity + variables: + - name: stationPressure + obs prior filters: # Initial Error Assignments for SFC Observations - filter: Perform Action @@ -103,36 +108,107 @@ obs prior filters: xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] errors: [ 110, 120, 120, 120, 120, 1.0e+11] -obs post filters: -# Observation range sanity check -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject +# Create PreQC group variable (pqm in GSI read_prepbufr) +- filter: Variable Assignment + assignments: + - name: InputObsError/stationPressure + type: float + source variable: ObsErrorData/stationPressure -# Reject all ObsType 183 -- filter: RejectList +# Set observation quality-realted variables +# Create PreQC group variable (pqm in GSI read_prepbufr) +- filter: Variable Assignment + assignments: + - name: PreQC/stationPressure + type: int + source variable: QualityMarker/stationPressure + +# Create PreUseFlag group variable (usage in GSI read_prepbufr) +# Initialize +- filter: Variable Assignment + assignments: + - name: PreUseFlag/stationPressure + type: int + source variable: PreQC/stationPressure + +- filter: Variable Assignment + where: + - variable: + name: PreUseFlag/stationPressure + is_in: 1-15 + assignments: + - name: PreUseFlag/stationPressure + value: 0 +# Re-assignment +- filter: Variable Assignment where: - variable: name: ObsType/stationPressure is_in: 183 + assignments: + - name: PreUseFlag/stationPressure + value: 100 -# Reject surface pressure below 500 hPa -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject +- filter: Variable Assignment + where: + - variable: + name: ObsValue/stationPressure + is_defined: + - variable: + name: ObsValue/stationPressure + maxvalue: 50000.00 + where operator: and + assignments: + - name: PreUseFlag/stationPressure + value: 100 -- filter: RejectList +- filter: Variable Assignment where: - variable: name: PreQC/stationPressure - is_in: 4-15 + is_in: 9, 12, 15 + assignments: + - name: PreUseFlag/stationPressure + value: 100 + +- filter: Variable Assignment + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + assignments: + - name: PreUseFlag/stationPressure + value: 101 + +## Observation range sanity check +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 37499.0 +# maxvalue: 106999.0 +# action: +# name: reject + +## Reject all ObsType 183 +#- filter: RejectList +# where: +# - variable: +# name: ObsType/stationPressure +# is_in: 183 + +## Reject surface pressure below 500 hPa +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 50000.00 +# action: +# name: reject + +#- filter: RejectList +# where: +# - variable: +# name: PreQC/stationPressure +# is_in: 4-15 # Inflate obs error based on obs type - filter: Perform Action @@ -145,6 +221,7 @@ obs post filters: name: inflate error inflation factor: 1.2 +obs post filters: # Calculate obs error inflation factors for duplicated observations at the same location - filter: Variable Assignment assignments: @@ -172,26 +249,6 @@ obs post filters: name: inflate error inflation factor: 0.7 -# Reduce original observation error based on obs type and subtype -# In this case: reduce original obs error for buoy -- filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - # Calculate obs error inflation factors for large discrepancies between model and observations - filter: Variable Assignment assignments: @@ -290,6 +347,5 @@ obs post filters: is_not_in: 0, 1 action: name: reject - # End of Filters diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml b/parm/atm/obs/lists/gdas_prototype_3d.yaml index b32d4b634..bcb6ddb55 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml @@ -17,4 +17,5 @@ observers: ##- !INC ${OBS_YAML_DIR}/iasi_metop-b.yaml ##- !INC ${OBS_YAML_DIR}/sfc.yaml ##- !INC ${OBS_YAML_DIR}/sfcship.yaml +##- !INC ${OBS_YAML_DIR}/conv_ps.yaml ##- !INC ${OBS_YAML_DIR}/gnssro.yaml diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml index 20a449a8a..6c795e6c5 100644 --- a/parm/atm/obs/testing/conv_ps.yaml +++ b/parm/atm/obs/testing/conv_ps.yaml @@ -104,15 +104,14 @@ obs prior filters: xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] errors: [ 110, 120, 120, 120, 120, 1.0e+11] -obs post filters: -# Observation range sanity check -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject +## Observation range sanity check +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 37499.0 +# maxvalue: 106999.0 +# action: +# name: reject # Reject all ObsType 183 - filter: RejectList @@ -121,19 +120,19 @@ obs post filters: name: ObsType/stationPressure is_in: 183 -# Reject surface pressure below 500 hPa -- filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject +## Reject surface pressure below 500 hPa +#- filter: Bounds Check +# filter variables: +# - name: stationPressure +# minvalue: 50000.00 +# action: +# name: reject -- filter: RejectList - where: - - variable: - name: PreQC/stationPressure - is_in: 4-15 +#- filter: RejectList +# where: +# - variable: +# name: PreQC/stationPressure +# is_in: 4-15 # Inflate obs error based on obs type - filter: Perform Action @@ -146,6 +145,7 @@ obs post filters: name: inflate error inflation factor: 1.2 +obs post filters: # Calculate obs error inflation factors for duplicated observations at the same location - filter: Variable Assignment assignments: @@ -173,26 +173,6 @@ obs post filters: name: inflate error inflation factor: 0.7 -# Reduce original observation error based on obs type and subtype -# In this case: reduce original obs error for buoy -- filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - # Calculate obs error inflation factors for large discrepancies between model and observations - filter: Variable Assignment assignments: diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 64f8ec8c0..a9258d744 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -3,6 +3,7 @@ cost function: time window: begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' length: $(ATM_WINDOW_LENGTH) + bound to include: begin analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: fms initialization: