diff --git a/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml b/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml new file mode 100644 index 00000000..5a39f5af --- /dev/null +++ b/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml @@ -0,0 +1,582 @@ + - obs space: + name: atms_n20 + distribution: + name: "@DISTRIBUTION@" + obsdatain: + engine: + type: H5File + obsfile: data/obs/ioda_atms_n20.nc + obsdataout: + engine: + type: H5File + obsfile: jdiag_atms_n20.nc4 + simulated variables: [brightnessTemperature] + observed variables: [brightnessTemperature] + channels: &atms_n20_channels 1-22 + obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + Cloud_Seeding: true + SurfaceWindGeoVars: uv + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: data/crtm/ + IRVISlandCoeff: IGBP + linear obs operator: + Absorbers: [H2O, O3] + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 300e3 # orig + obs bias: + input file: data/obs/atms_n20.satbias.nc4 + output file: out_atms_n20.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &atms_n20_tlapse data/obs/atms_n20.tlapse.txt + - name: lapseRate + tlapse: *atms_n20_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: data/obs/atms_n20.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: out_atms_n20.satbias_cov.nc4 + + obs pre filters: + # Step 0-A: Create Diagnostic Flags + - filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + obs post filters: + # Step 0-B: Calculate derived variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + + # Calculate Innovation@DerivedMetaData + - filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_n20_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_n20_channels + - name: brightnessTemperature@HofX + channels: *atms_n20_channels + coefs: [1, -1] + + - filter: Bounds Check + filter variables: + #- name: brightnessTemperature + - name: HofX/brightnessTemperature + channels: *atms_n20_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject + + # Regional Domain Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: GeoVaLs/observable_domain_mask + flag all filter variables if any test variable is out of bounds: true + minvalue: 0.0 + maxvalue: 0.5 + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: data Thinning + - filter: Gaussian Thinning + #horizontal_mesh: 145 + horizontal_mesh: 60 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + + # Step 10: Gross check + # Remove data if abs(Obs-HofX) > absolute threhold + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_n20_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use passive_bc: true + sensor: atms_n20 + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, -1, -1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, -1, -1, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject