From bb5428d6bca35efb6e75264e3c0f6b430ab564a1 Mon Sep 17 00:00:00 2001 From: Xin Jin Date: Tue, 4 Jun 2024 11:23:48 -0500 Subject: [PATCH] save --- parm/atm/obs/config/ssmis_f17.yaml | 770 +++++++++++++++++++++++ parm/atm/obs/config/ssmis_f18.yaml | 770 +++++++++++++++++++++++ parm/ioda/bufr2ioda/bufr2ioda_ssmis.yaml | 123 ++++ ush/ioda/bufr2ioda/bufr2ioda_ssmis.json | 18 + ush/ioda/bufr2ioda/bufr2ioda_ssmis.py | 226 +++++++ 5 files changed, 1907 insertions(+) create mode 100644 parm/atm/obs/config/ssmis_f17.yaml create mode 100644 parm/atm/obs/config/ssmis_f18.yaml create mode 100644 parm/ioda/bufr2ioda/bufr2ioda_ssmis.yaml create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_ssmis.json create mode 100644 ush/ioda/bufr2ioda/bufr2ioda_ssmis.py diff --git a/parm/atm/obs/config/ssmis_f17.yaml b/parm/atm/obs/config/ssmis_f17.yaml new file mode 100644 index 000000000..bfab30c20 --- /dev/null +++ b/parm/atm/obs/config/ssmis_f17.yaml @@ -0,0 +1,770 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID ssmis_f17 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: ssmis_f17 + obsdatain: + engine: + type: H5File + obsfile: !ENV ssmis_f17_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ssmis_f17_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-24 +geovals: + filename: !ENV ssmis_f17_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV ssmis_f17_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: cloud_liquid_water + sensor: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 + - name: cosine_of_latitude_times_orbit_node + - name: sine_of_latitude + - name: lapse_rate + order: 2 + tlapse: !ENV &ssmis_f17_tlap ssmis_f17_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *ssmis_f17_tlap + - name: emissivity + - name: scan_angle + var_name: sensorScanPosition + order: 4 + - name: scan_angle + var_name: sensorScanPosition + order: 3 + - name: scan_angle + var_name: sensorScanPosition + order: 2 + - name: scan_angle + var_name: sensorScanPosition + +obs prior filters: +# Step 1: Initial Observation Error Assignment +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] +obs post filters: +# Step 2: Gross check over water-dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# Step 3: Reject all channels if surface height is greater than 2km +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *all_channels + where: + - variable: + name: GeoVaLs/surface_geopotential_height + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 4: Reject data over mixed surface type +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 5: Channel 2 O-F check over non-water dominant area +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 6: Gross check over non-water dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + +# Step 7: Scattering check for channels 9-11 using channels 8 and 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.485934, 0.485934, 0.473806, -0.473806] + intercept: 271.252327 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.413688, 0.413688, 0.361549, -0.361549] + intercept: 272.280341 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.400882, 0.400882, 0.270510, -0.270510] + intercept: 278.824902 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: DerivedMetaData/SSMISScatteringIndex9 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: DerivedMetaData/SSMISScatteringIndex10 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: DerivedMetaData/SSMISScatteringIndex11 + maxvalue: 2 + +# Step 8: NSST retrieval check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1] + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Error inflation based on surface jacobian +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] + obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 10: Final gross check +# Channel 1 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: [ 2.25 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + +# Channel 2 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + +# Channel 3 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + +# Channel 4 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + +# Channel 5 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + +# Channel 6 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + +# Channel 7 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + +# Channel 8 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + +# Channel 9 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + +# Channel 10 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + +# Channel 11 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + +# Channel 12 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: [ 3.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + +# Channel 13 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: [ 1.905 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + +# Channel 14 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: [ 2.16 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + +# Channel 15 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + +# Channel 16 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: [ 2.01 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + +# Channel 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: [ 2.61 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + +# Channel 18 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: [ 5.625 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + +# Channel 19 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + +# Channel 20 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + +# Channel 21 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: [ 3.00 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + +# Channel 22 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: [ 9.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + +# Channel 23 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + +# Channel 24 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + +passedBenchmark: 126336 diff --git a/parm/atm/obs/config/ssmis_f18.yaml b/parm/atm/obs/config/ssmis_f18.yaml new file mode 100644 index 000000000..87859a26c --- /dev/null +++ b/parm/atm/obs/config/ssmis_f18.yaml @@ -0,0 +1,770 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID ssmis_f18 + EndianType: little_endian + CoefficientPath: crtm/ +obs space: + name: ssmis_f18 + obsdatain: + engine: + type: H5File + obsfile: !ENV ssmis_f18_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV ssmis_f18_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &all_channels 1-24 +geovals: + filename: !ENV ssmis_f18_geoval_${CDATE}.nc4 +obs bias: + input file: !ENV ssmis_f18_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: cloud_liquid_water + sensor: SSMIS + ch19h: 12 + ch19v: 13 + ch22v: 14 + ch37h: 15 + ch37v: 16 + ch91v: 17 + ch91h: 18 + - name: cosine_of_latitude_times_orbit_node + - name: sine_of_latitude + - name: lapse_rate + order: 2 + tlapse: !ENV &ssmis_f18_tlap ssmis_f18_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *ssmis_f18_tlap + - name: emissivity + - name: scan_angle + var_name: sensorScanPosition + order: 4 + - name: scan_angle + var_name: sensorScanPosition + order: 3 + - name: scan_angle + var_name: sensorScanPosition + order: 2 + - name: scan_angle + var_name: sensorScanPosition + +obs prior filters: +# Step 1: Initial Observation Error Assignment +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: assign error + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] +obs post filters: +# Step 2: Gross check over water-dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + +# Step 3: Reject all channels if surface height is greater than 2km +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *all_channels + where: + - variable: + name: GeoVaLs/surface_geopotential_height + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 4: Reject data over mixed surface type +- filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 5: Channel 2 O-F check over non-water dominant area +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 6: Gross check over non-water dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + +# Step 7: Scattering check for channels 9-11 using channels 8 and 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.485934, 0.485934, 0.473806, -0.473806] + intercept: 271.252327 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.413688, 0.413688, 0.361549, -0.361549] + intercept: 272.280341 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SSMISScatteringIndex11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.400882, 0.400882, 0.270510, -0.270510] + intercept: 278.824902 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: DerivedMetaData/SSMISScatteringIndex9 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: DerivedMetaData/SSMISScatteringIndex10 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: DerivedMetaData/SSMISScatteringIndex11 + maxvalue: 2 + +# Step 8: NSST retrieval check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *all_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *all_channels + options: + channels: *all_channels + use_flag: [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1] + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Error inflation based on surface jacobian +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *all_channels + options: + sensor: *Sensor_ID + channels: *all_channels + obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] + obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *all_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *all_channels + +# Step 10: Final gross check +# Channel 1 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: [ 2.25 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + +# Channel 2 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + +# Channel 3 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + +# Channel 4 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + +# Channel 5 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + +# Channel 6 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + +# Channel 7 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + +# Channel 8 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + +# Channel 9 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + +# Channel 10 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + +# Channel 11 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + +# Channel 12 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: [ 3.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + +# Channel 13 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: [ 1.905 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + +# Channel 14 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: [ 2.16 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + +# Channel 15 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + +# Channel 16 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: [ 2.01 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + +# Channel 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: [ 2.61 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + +# Channel 18 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: [ 5.625 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + +# Channel 19 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + +# Channel 20 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + +# Channel 21 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: [ 3.00 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + +# Channel 22 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: [ 9.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + +# Channel 23 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + +# Channel 24 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + +passedBenchmark: 72466 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ssmis.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ssmis.yaml new file mode 100644 index 000000000..c01d76432 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ssmis.yaml @@ -0,0 +1,123 @@ +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.ssmisu.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + scanLineNumber: + query: "*/SLNM" + + # The type should be integer. + # However, UFO/CRTM requred this variable to be float + fieldOfViewNumber: + query: "*/FOVN" +# type: float + + surfaceFlag: + query: "*/SFLG" + + rainFlag: + query: "*/RFLAG" + + orbitNumber: + query: "*/ORBN" + + sensorChannelNumber: + query: '*/SSMISCHN/CHNM' + + # ObsValue + # Remapped Brightness Temperature + remappedBT: + remappedBrightnessTemperature: + fieldOfViewNumber: "*/FOVN" + sensorChannelNumber: "*/SSMISCHN/CHNM" + brightnessTemperature: "*/SSMISCHN/TMBR" + obsTime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _249: f16 + _285: f17 + _286: f18 + _287: f19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.ssmis_$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/SSMISCHN" + + globals: + - name: "platformCommonName" + type: string + value: "SSMIS" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-203 SSMIS ATENNA/BRIGHTNESS TEMPERATURE DATA" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + # Remapped Brightness Temperature + - name: "ObsValue/brightnessTemperature" + source: variables/remappedBT + longName: "3-by-3 Averaged Brightness Temperature" + units: "K" + chunks: [10000, 22] diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.json b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.json new file mode 100644 index 000000000..6599eb02d --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.json @@ -0,0 +1,18 @@ +{ + "data_format" : "bufr_d", + "data_type" : "ssmis", + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005044", "NC005045", "NC005046" ], + "data_description" : "NC005044 JMA SATWIND, AHI HIM8 IR(LW)/VIS/WV-CT/WV-CS; NC005045 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS; NC005046 JMA SATWIND, HIMAWARI-8 IR(LW)/VIS/WV-CT/WV-CS", + "data_provider" : "JMA", + "sensor_info" : { "sensor_name": "AHI", "sensor_full_name": "Advanced Himawari Imager", "sensor_id": 999 }, + "satellite_info" : [ + { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 249 }, + { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 285 }, + { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 286 }, + { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 287 } + ] +} diff --git a/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py new file mode 100644 index 000000000..29b59bdac --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_ssmis.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +import argparse +import numpy as np +import yaml +import numpy.ma as ma +from pyiodaconv import bufr +import calendar +import json +import time +import math +import datetime +import os +from datetime import datetime +from pyioda import ioda_obs_space as ioda_ospace +from wxflow import Logger + +from pprint import pprint + +# ==================================================================== +# Satellite Winds (AMV) BUFR dump file for AHI/Himawari +# ==================================================================== +# All subsets contain all spectral bands: NC005044 NC005045 NC005046 +# ==================================================================== +# Spectral Band | Code (002023) | ObsType +# -------------------------------------------------------------------- +# IRLW (Freq < 5E+13) | Method 1 | 252 +# VIS | Method 2 | 242 +# WV Cloud Top | Method 3 | 250 +# WV Clear Sky/ Deep Layer | Method 5 | 250 +# ==================================================================== + +# Define and initialize global variables +global float32_fill_value +global int32_fill_value +global int64_fill_value + +float32_fill_value = np.float32(0) +int32_fill_value = np.int32(0) +int64_fill_value = np.int64(0) + + + + +def get_config(config, logger): + + subsets = config["subsets"] + logger.debug(f"Checking subsets = {subsets}") + + # Get parameters from configuration + subsets = config["subsets"] + data_format = config["data_format"] + data_type = config["data_type"] + data_description = config["data_description"] + data_provider = config["data_provider"] + cycle_type = config["cycle_type"] + dump_dir = config["dump_directory"] + ioda_dir = config["ioda_directory"] + cycle = config["cycle_datetime"] + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + + satellite_info_array = config["satellite_info"] + sensor_name = config["sensor_info"]["sensor_name"] + sensor_full_name = config["sensor_info"]["sensor_full_name"] + sensor_id = config["sensor_info"]["sensor_id"] + + # Get derived parameters + yyyymmdd = cycle[0:8] + hh = cycle[8:10] + reference_time = datetime.strptime(cycle, "%Y%m%d%H") + reference_time = reference_time.strftime("%Y-%m-%dT%H:%M:%SZ") + + # General informaton + converter = 'BUFR to IODA Converter' + process_level = 'Level-2' + platform_description = 'Himawari-8' + sensor_description = 'Advanced Himawari Imager' + + logger.info(f'sensor_name = {sensor_name}') + logger.info(f'sensor_full_name = {sensor_full_name}') + logger.info(f'sensor_id = {sensor_id}') + logger.info(f'reference_time = {reference_time}') + + bufrfile = f"{cycle_type}.t{hh}z.{data_type}.tm00.{data_format}" + DATA_PATH = os.path.join(dump_dir, f"{cycle_type}.{yyyymmdd}", str(hh), 'atmos', bufrfile) + if not os.path.isfile(DATA_PATH): + logger.info(f"DATA_PATH {DATA_PATH} does not exist") + return + logger.debug(f"The DATA_PATH is: {DATA_PATH}") + + + +def get_result_set(data_path, q): + logger.info('Executing QuerySet to get ResultSet') + with bufr.File(data_path) as f: + try: + r = f.execute(q) + except Exception as err: + logger.info(f'Return with {err}') + return + + # MetaData + satid = r.get('satelliteId') + year = r.get('year') + month = r.get('month') + day = r.get('day') + hour = r.get('hour') + minute = r.get('minute') + second = r.get('second') + lat = r.get('latitude') + lon = r.get('longitude') + satzenang = r.get('satelliteZenithAngle') + pressure = r.get('pressure', type='float') + chanfreq = r.get('sensorCentralFrequency', type='float') + + # Processing Center + ogce = r.get('dataProviderOrigin') + ga = r.get('windGeneratingApplication') + + # Quality Information + qi = r.get('qualityInformationWithoutForecast', type='float') + # For AHI/Himawari data, qi w/o forecast (qifn) is packaged in same + # vector where ga == 102. Must conduct a search and extract the + # correct vector for gnap and qi + # 1. Find dimension-sizes of ga and qi (should be the same!) + gDim1, gDim2 = np.shape(ga) + qDim1, qDim2 = np.shape(qi) + logger.info(f'Generating Application and Quality Information SEARCH:') + logger.info(f'Dimension size of GNAP ({gDim1},{gDim2})') + logger.info(f'Dimension size of PCCF ({qDim1},{qDim2})') + # 2. Initialize gnap and qifn as None, and search for dimension of + # ga with values of 102. If the same column exists for qi, assign + # gnap to ga[:,i] and qifn to qi[:,i], else raise warning that no + # appropriate GNAP/PCCF combination was found + gnap = None + qifn = None + for i in range(gDim2): + if np.unique(ga[:, i].squeeze()) == 102: + if i <= qDim2: + logger.info(f'GNAP/PCCF found for column {i}') + gnap = ga[:, i].squeeze() + qifn = qi[:, i].squeeze() + else: + logger.info(f'ERROR: GNAP column {i} outside of PCCF dimension {qDim2}') + if (gnap is None) & (qifn is None): + logger.info(f'ERROR: GNAP == 102 NOT FOUND OR OUT OF PCCF DIMENSION-RANGE, WILL FAIL!') + + # Wind Retrieval Method Information + swcm = r.get('windComputationMethod') + + # ObsValue + # Wind direction and Speed + wdir = r.get('windDirection', type='float') + wspd = r.get('windSpeed') + + # DateTime: seconds since Epoch time + # IODA has no support for numpy datetime arrays dtype=datetime64[s] + timestamp = r.get_datetime('year', 'month', 'day', 'hour', 'minute', 'second').astype(np.int64) + + # Check BUFR variable generic dimension and type + + # Global variables declaration + # Set global fill values + float32_fill_value = satzenang.fill_value + int32_fill_value = satid.fill_value + int64_fill_value = timestamp.fill_value.astype(np.int64) + + + + # ========================= + # Create derived variables + # ========================= + start_time = time.time() + + logger.info('Creating derived variables') + logger.debug('Creating derived variables - wind components (uob and vob)') + + + height = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + stnelev = np.full_like(pressure, fill_value=pressure.fill_value, dtype=np.float32) + + end_time = time.time() + running_time = end_time - start_time + logger.info(f'Processing time for creating derived variables : {running_time} seconds') + + # ===================================== + # Split output based on satellite id + # Create IODA ObsSpace + # Write IODA output + # ===================================== + logger.info('Create IODA ObsSpace and Write IODA output based on satellite ID') + + # Find unique satellite identifiers in data to process + unique_satids = np.unique(satid) + logger.info(f'Number of Unique satellite identifiers: {len(unique_satids)}') + logger.info(f'Unique satellite identifiers: {unique_satids}') + + logger.debug(f'Loop through unique satellite identifier {unique_satids}') + + logger.info("All Done!") + + +def get_yaml_config(yaml_file): + with open(yaml_file, 'r') as file: + yaml_config = yaml.load(file, Loader=yaml.FullLoader) + pprint(yaml_config) + return yaml_config + + +if __name__ == '__main__': + + + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger(os.path.basename(__file__), level=log_level, colored_log=True) + + yaml_config = get_yaml_config(args.config) + + get_config(yaml_config, logger) + +