diff --git a/dump/aux/amsua_metop-a.ACCoeff.nc b/dump/aux/amsua_metop-a.ACCoeff.nc new file mode 100755 index 0000000..28b062a Binary files /dev/null and b/dump/aux/amsua_metop-a.ACCoeff.nc differ diff --git a/dump/aux/amsua_metop-b.ACCoeff.nc b/dump/aux/amsua_metop-b.ACCoeff.nc new file mode 100755 index 0000000..fdc91fc Binary files /dev/null and b/dump/aux/amsua_metop-b.ACCoeff.nc differ diff --git a/dump/aux/amsua_metop-c.ACCoeff.nc b/dump/aux/amsua_metop-c.ACCoeff.nc new file mode 100755 index 0000000..c7a3115 Binary files /dev/null and b/dump/aux/amsua_metop-c.ACCoeff.nc differ diff --git a/dump/aux/amsua_n15.ACCoeff.nc b/dump/aux/amsua_n15.ACCoeff.nc new file mode 100755 index 0000000..bf3a9c0 Binary files /dev/null and b/dump/aux/amsua_n15.ACCoeff.nc differ diff --git a/dump/aux/amsua_n16.ACCoeff.nc b/dump/aux/amsua_n16.ACCoeff.nc new file mode 100755 index 0000000..efeef51 Binary files /dev/null and b/dump/aux/amsua_n16.ACCoeff.nc differ diff --git a/dump/aux/amsua_n17.ACCoeff.nc b/dump/aux/amsua_n17.ACCoeff.nc new file mode 100755 index 0000000..a50cc26 Binary files /dev/null and b/dump/aux/amsua_n17.ACCoeff.nc differ diff --git a/dump/aux/amsua_n18.ACCoeff.nc b/dump/aux/amsua_n18.ACCoeff.nc new file mode 100755 index 0000000..7231090 Binary files /dev/null and b/dump/aux/amsua_n18.ACCoeff.nc differ diff --git a/dump/aux/amsua_n19.ACCoeff.nc b/dump/aux/amsua_n19.ACCoeff.nc new file mode 100755 index 0000000..bc36197 Binary files /dev/null and b/dump/aux/amsua_n19.ACCoeff.nc differ diff --git a/dump/config/bufr_bufr_backend_amsua.yaml b/dump/config/bufr_bufr_backend_amsua.yaml new file mode 100644 index 0000000..ed97321 --- /dev/null +++ b/dump/config/bufr_bufr_backend_amsua.yaml @@ -0,0 +1,39 @@ +time window: + begin: "2018-04-14T21:00:00Z" + end: "2023-12-15T03:00:00Z" + +observations: +- obs space: + name: "1bamua" + simulated variables: ['brightnessTemperature'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.1bamua.tm00.bufr_d" + mapping file: "./bufr2ioda_1bamua_mapping.yaml" + category: ["n19"] # optional (needed if the BUFR mapping defines splits) + cache categories: # optional + - ["n19"] + - ["n20"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.1bamua_n19.tm00.nc" + +- obs space: + name: "esamua_n19" + simulated variables: ['brightnessTemperature'] + obsdatain: + engine: + type: bufr + obsfile: "./testinput/2021080100/gdas.t00z.esamua.tm00.bufr_d" + mapping file: "./bufr2ioda_esamua_mapping.yaml" + category: ["n19"] # optional (needed if the BUFR mapping defines splits) + cache categories: # optional + - ["n19"] + - ["n20"] + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.esamua_n19.tm00.nc" + diff --git a/dump/config/bufr_script_backend_amsua.yaml b/dump/config/bufr_script_backend_amsua.yaml new file mode 100644 index 0000000..f81f2c0 --- /dev/null +++ b/dump/config/bufr_script_backend_amsua.yaml @@ -0,0 +1,22 @@ +time window: + begin: "2020-01-14T21:00:00Z" + end: "2024-12-15T03:00:00Z" + +observations: + - obs space: + name: "amsua_metop-c" + simulated variables: ["brightnessTemperature"] + obsdatain: + engine: + type: script + script file: "bufr2ioda_amsua.py" + args: + input_path1: "./testinput/2021080100/gdas.t00z.esamua.tm00.bufr_d" + input_path2: "./testinput/2021080100/gdas.t00z.1bamua.tm00.bufr_d" + yaml_1b: "../../dump/mapping/bufr2ioda_1bamua_mapping.yaml" + yaml_es: "../../dump/mapping/bufr2ioda_esamua_mapping.yaml" + category: "metop-c" + obsdataout: + engine: + type: H5File + obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.amsua_metop-c.tm00.nc" diff --git a/dump/mapping/bufr2ioda_1bamua_mapping.yaml b/dump/mapping/bufr2ioda_1bamua_mapping.yaml new file mode 100644 index 0000000..ab4c0a1 --- /dev/null +++ b/dump/mapping/bufr2ioda_1bamua_mapping.yaml @@ -0,0 +1,134 @@ +bufr: + splits: + satId: + category: + map: + _209: n18 + _223: n19 + _3: metop-b + _4: metop-a + _5: metop-c + variable: satelliteIdentifier + variables: + brightnessTemperature: + query: '*/BRITCSTC/TMBR' + fieldOfViewNumber: + query: '*/FOVN' + heightOfStation: + query: '*/HMSL' + latitude: + query: '*/CLAT' + longitude: + query: '*/CLON' + satelliteIdentifier: + query: '*/SAID' + sensorAzimuthAngle: + query: '*/BEARAZ' + sensorChannelNumber: + query: '*/BRITCSTC/CHNM' + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: '*/FOVN' + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + sensorZenithAngle: + query: '*/SAZA' + solarAzimuthAngle: + query: '*/SOLAZI' + solarZenithAngle: + query: '*/SOZA' + timestamp: + datetime: + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + month: '*/MNTH' + second: '*/SECO' + year: '*/YEAR' +encoder: + backend: netcdf + dimensions: + - name: Channel + path: '*/BRITCSTC' + source: variables/sensorChannelNumber + globals: + - name: platformCommonName + type: string + value: AMSUA + - name: platformLongDescription + type: string + value: MTYP 021-023 PROC AMSU-A 1B Tb + variables: + - longName: Satellite Identifier + name: MetaData/satelliteIdentifier + source: variables/satelliteIdentifier + - longName: Field Of View Number + name: MetaData/sensorScanPosition + source: variables/fieldOfViewNumber + - longName: Altitude of Satellite + name: MetaData/heightOfStation + source: variables/heightOfStation + units: m + - longName: Solar Zenith Angle + name: MetaData/solarZenithAngle + range: + - 0 + - 180 + source: variables/solarZenithAngle + units: degree + - longName: Solar Azimuth Angle + name: MetaData/solarAzimuthAngle + range: + - 0 + - 360 + source: variables/solarAzimuthAngle + units: degree + - longName: Sensor Zenith Angle + name: MetaData/sensorZenithAngle + range: + - 0 + - 90 + source: variables/sensorZenithAngle + units: degree + - longName: Sensor Azimuth Angle + name: MetaData/sensorAzimuthAngle + range: + - 0 + - 360 + source: variables/sensorAzimuthAngle + units: degree + - longName: Sensor View Angle + name: MetaData/sensorViewAngle + source: variables/sensorScanAngle + units: degree + - longName: Sensor Channel Number + name: MetaData/sensorChannelNumber + source: variables/sensorChannelNumber + - chunks: + - 1000 + - 15 + compressionLevel: 4 + coordinates: longitude latitude Channel + longName: Brightness Temperature + name: ObsValue/brightnessTemperature + range: + - 100 + - 500 + source: variables/brightnessTemperature + units: K + - longName: Datetime + name: MetaData/dateTime + source: variables/timestamp + units: seconds since 1970-01-01T00:00:00Z + - longName: Latitude + name: MetaData/latitude + range: + - -90 + - 90 + source: variables/latitude + units: degree_north + - longName: Longitude + name: MetaData/longitude + source: variables/longitude + units: degree_east diff --git a/dump/mapping/bufr2ioda_amsua.py b/dump/mapping/bufr2ioda_amsua.py new file mode 100644 index 0000000..44114a9 --- /dev/null +++ b/dump/mapping/bufr2ioda_amsua.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +import bufr +import netCDF4 as nc +import os +from pyioda.ioda.Engines.Bufr import Encoder +from wxflow import Logger + +logger = Logger(os.path.basename(__file__), level='INFO', colored_log=True) + +YAML_NORMAL = False # current as normal need remap for path2/1bama + +R1000 = 1000.0 +R1000000 = 1000000.0 +INVALID = R1000 +# Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE +# Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520 +TSPACE = 2.7253 + +nc_dir = '../../dump/aux' + + +class ACCoeff: + def __init__(self, ac_dir, sat_id='n19'): + file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '.ACCoeff.nc') + nc_file = nc.Dataset(file_name) + self.n_fovs = len(nc_file.dimensions['n_FOVs']) + self.n_channels = len(nc_file.dimensions['n_Channels']) + self.a_earth = nc_file.variables['A_earth'][:] + self.a_platform = nc_file.variables['A_platform'][:] + self.a_space = nc_file.variables['A_space'][:] + self.a_ep = self.a_earth + self.a_platform + self.a_sp = self.a_space * TSPACE + + +def remove_ant_corr(i, ac, ifov, t): + # AC: Structure containing the antenna correction coefficients for the sensor of interest. + # iFOV: The FOV index for a scanline of the sensor of interest. + # T: On input, this argument contains the brightness + + t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov] + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t + + +def apply_ant_corr(i, ac, ifov, t): + # t: on input, this argument contains the antenna temperatures for the sensor channels. + t = (t - ac.a_sp[i, ifov]) / ac.a_ep[i, ifov] + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t + + +def get_description(yaml_path): + description = bufr.encoders.Description(yaml_path) + return description + + +def apply_corr(sat_id, ta, ifov): + ac = ACCoeff(nc_dir, sat_id=sat_id) + if sat_id not in ['n15', 'n16']: + # Convert antenna temperature to brightness temperature + ifov = ifov.astype(int) - 1 + for i in range(ta.shape[1]): + logger.debug(f'inside loop for allpy ta to tb: i = {i}') + x = ta[:, i] + if YAML_NORMAL: + x = apply_ant_corr(i, ac, ifov, x) + else: + x = remove_ant_corr(i, ac, ifov, x) + x[x >= R1000] = R1000000 + ta[:, i] = x + return ta + + +def re_map_variable(container): + # read_bufrtovs.f90 + # antcorr_application.f90 + # search the keyword “ta2tb” for details + sat_ids = container.all_sub_categories() + for sat_id in sat_ids: + logger.info(f'Converting for {sat_id[0]}, ...') + ta = container.get('variables/brightnessTemperature', sat_id) + if ta.shape[0]: + ifov = container.get('variables/fieldOfViewNumber', sat_id) + tb = apply_corr(sat_id[0], ta, ifov) + container.replace('variables/brightnessTemperature', tb, sat_id) + + +def get_one_data(input_path, yaml_path, category): + cache = bufr.DataCache.has(input_path, yaml_path) + if cache: + logger.info(f'The cache existed get data container from it') + container = bufr.DataCache.get(input_path, yaml_path) + else: + # If cacache does not exist, get data into cache + # Get data info container first + logger.info(f'The cache is not existed') + container = bufr.Parser(input_path, yaml_path).parse() + return cache, container + + +def mark_one_data(cache, input_path, yaml_path, category, container=None): + if cache: + logger.info(f'The cache existed get data container from it') + bufr.DataCache.mark_finished(input_path, yaml_path, [category]) + else: + logger.info(f'add original container list into a cache = {container.list()}') + bufr.DataCache.add(input_path, yaml_path, container.all_sub_categories(), container) + bufr.DataCache.mark_finished(input_path, yaml_path, [category]) + + +def create_obs_group(input_path1, input_path2, yaml_1b, yaml_es, category): + logger.info(f'imput_path: {input_path1}, {input_path2}, and category: {category}') + logger.info(f'Entering function to create obs group for {category} with yaml path {yaml_es} and {yaml_1b}') + cache_1, container_1 = get_one_data(input_path1, yaml_es, category) + cache_2, container_2 = get_one_data(input_path2, yaml_1b, category) + + re_map_variable(container_2) + + container = container_1 + container.append(container_2) + + logger.info('Container append done') + data = Encoder(get_description(yaml_es)).encode(container)[(category,)] + mark_one_data(cache_1, input_path1, yaml_es, category, container=container_1) + mark_one_data(cache_2, input_path2, yaml_1b, category, container=container_2) + return data + diff --git a/dump/mapping/bufr2ioda_esamua_mapping.yaml b/dump/mapping/bufr2ioda_esamua_mapping.yaml new file mode 100644 index 0000000..b3d251c --- /dev/null +++ b/dump/mapping/bufr2ioda_esamua_mapping.yaml @@ -0,0 +1,136 @@ +bufr: + splits: + satId: + category: + map: + _209: n18 + _223: n19 + _3: metop-b + _4: metop-a + _5: metop-c + variable: satelliteIdentifier + variables: + brightnessTemperature: + query: '*/ATNCHV/TMBRST' + fieldOfViewNumber: + query: '*/FOVN' + heightOfStation: + query: '*/SELV' + latitude: + query: '*/CLATH' + longitude: + query: '*/CLONH' + satelliteIdentifier: + query: '*/SAID' + sensorAzimuthAngle: + query: '*/BEARAZ' + sensorChannelNumber: + query: '*/ATNCHV/INCN' + transforms: + - offset: -27 + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: '*/FOVN' + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + sensorZenithAngle: + query: '*/SAZA' + solarAzimuthAngle: + query: '*/SOLAZI' + solarZenithAngle: + query: '*/SOZA' + timestamp: + datetime: + day: '*/DAYS' + hour: '*/HOUR' + minute: '*/MINU' + month: '*/MNTH' + second: '*/SECO' + year: '*/YEAR' +encoder: + backend: netcdf + dimensions: + - name: Channel + path: '*/ATNCHV' + source: variables/sensorChannelNumber + globals: + - name: platformCommonName + type: string + value: AMSUA + - name: platformLongDescription + type: string + value: MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA) + variables: + - longName: Satellite Identifier + name: MetaData/satelliteIdentifier + source: variables/satelliteIdentifier + - longName: Field Of View Number + name: MetaData/sensorScanPosition + source: variables/fieldOfViewNumber + - longName: Altitude of Satellite + name: MetaData/heightOfStation + source: variables/heightOfStation + units: m + - longName: Solar Zenith Angle + name: MetaData/solarZenithAngle + range: + - 0 + - 180 + source: variables/solarZenithAngle + units: degree + - longName: Solar Azimuth Angle + name: MetaData/solarAzimuthAngle + range: + - 0 + - 360 + source: variables/solarAzimuthAngle + units: degree + - longName: Sensor Zenith Angle + name: MetaData/sensorZenithAngle + range: + - 0 + - 90 + source: variables/sensorZenithAngle + units: degree + - longName: Sensor Azimuth Angle + name: MetaData/sensorAzimuthAngle + range: + - 0 + - 360 + source: variables/sensorAzimuthAngle + units: degree + - longName: Sensor View Angle + name: MetaData/sensorViewAngle + source: variables/sensorScanAngle + units: degree + - longName: Sensor Channel Number + name: MetaData/sensorChannelNumber + source: variables/sensorChannelNumber + - chunks: + - 1000 + - 15 + compressionLevel: 4 + coordinates: longitude latitude Channel + longName: Brightness Temperature + name: ObsValue/brightnessTemperature + range: + - 100 + - 500 + source: variables/brightnessTemperature + units: K + - longName: Datetime + name: MetaData/dateTime + source: variables/timestamp + units: seconds since 1970-01-01T00:00:00Z + - longName: Latitude + name: MetaData/latitude + range: + - -90 + - 90 + source: variables/latitude + units: degree_north + - longName: Longitude + name: MetaData/longitude + source: variables/longitude + units: degree_east