add mapping files for amsua

dump/config/bufr_bufr_backend_amsua.yaml

@@ -0,0 +1,39 @@
+time window:
+  begin: "2018-04-14T21:00:00Z"
+  end: "2023-12-15T03:00:00Z"
+
+observations:
+- obs space:
+    name: "1bamua"
+    simulated variables: ['brightnessTemperature']
+    obsdatain:
+      engine:
+        type: bufr 
+        obsfile: "./testinput/2021080100/gdas.t00z.1bamua.tm00.bufr_d"
+        mapping file: "./bufr2ioda_1bamua_mapping.yaml"
+        category: ["n19"]            # optional (needed if the BUFR mapping defines splits)
+        cache categories:            # optional
+          - ["n19"]
+          - ["n20"]
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.1bamua_n19.tm00.nc"
+
+- obs space:
+    name: "esamua_n19"
+    simulated variables: ['brightnessTemperature']
+    obsdatain:
+      engine:
+        type: bufr
+        obsfile: "./testinput/2021080100/gdas.t00z.esamua.tm00.bufr_d"
+        mapping file: "./bufr2ioda_esamua_mapping.yaml"
+        category: ["n19"]            # optional (needed if the BUFR mapping defines splits)
+        cache categories:            # optional
+          - ["n19"]
+          - ["n20"]
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "./testoutput/2021080100/bufr_backend/gdas.t00z.esamua_n19.tm00.nc"
+

dump/config/bufr_script_backend_amsua.yaml

@@ -0,0 +1,22 @@
+time window:
+  begin: "2020-01-14T21:00:00Z"
+  end: "2024-12-15T03:00:00Z"
+
+observations:
+  - obs space:
+      name: "amsua_metop-c"
+      simulated variables: ["brightnessTemperature"]
+      obsdatain:
+        engine:
+          type: script
+          script file: "bufr2ioda_amsua.py"
+          args:
+            input_path1: "./testinput/2021080100/gdas.t00z.esamua.tm00.bufr_d"
+            input_path2: "./testinput/2021080100/gdas.t00z.1bamua.tm00.bufr_d"
+            yaml_1b: "../../dump/mapping/bufr2ioda_1bamua_mapping.yaml"
+            yaml_es: "../../dump/mapping/bufr2ioda_esamua_mapping.yaml" 
+            category: "metop-c"
+      obsdataout:
+        engine:
+          type: H5File
+          obsfile: "./testoutput/2021080100/script_backend/gdas.t00z.amsua_metop-c.tm00.nc"

dump/mapping/bufr2ioda_1bamua_mapping.yaml

@@ -0,0 +1,134 @@
+bufr:
+  splits:
+    satId:
+      category:
+        map:
+          _209: n18
+          _223: n19
+          _3: metop-b
+          _4: metop-a
+          _5: metop-c
+        variable: satelliteIdentifier
+  variables:
+    brightnessTemperature:
+      query: '*/BRITCSTC/TMBR'
+    fieldOfViewNumber:
+      query: '*/FOVN'
+    heightOfStation:
+      query: '*/HMSL'
+    latitude:
+      query: '*/CLAT'
+    longitude:
+      query: '*/CLON'
+    satelliteIdentifier:
+      query: '*/SAID'
+    sensorAzimuthAngle:
+      query: '*/BEARAZ'
+    sensorChannelNumber:
+      query: '*/BRITCSTC/CHNM'
+    sensorScanAngle:
+      sensorScanAngle:
+        fieldOfViewNumber: '*/FOVN'
+        scanStart: -48.333
+        scanStep: 3.333
+        sensor: amsua
+    sensorZenithAngle:
+      query: '*/SAZA'
+    solarAzimuthAngle:
+      query: '*/SOLAZI'
+    solarZenithAngle:
+      query: '*/SOZA'
+    timestamp:
+      datetime:
+        day: '*/DAYS'
+        hour: '*/HOUR'
+        minute: '*/MINU'
+        month: '*/MNTH'
+        second: '*/SECO'
+        year: '*/YEAR'
+encoder:
+  backend: netcdf
+  dimensions:
+  - name: Channel
+    path: '*/BRITCSTC'
+    source: variables/sensorChannelNumber
+  globals:
+  - name: platformCommonName
+    type: string
+    value: AMSUA
+  - name: platformLongDescription
+    type: string
+    value: MTYP 021-023 PROC AMSU-A 1B Tb
+  variables:
+  - longName: Satellite Identifier
+    name: MetaData/satelliteIdentifier
+    source: variables/satelliteIdentifier
+  - longName: Field Of View Number
+    name: MetaData/sensorScanPosition
+    source: variables/fieldOfViewNumber
+  - longName: Altitude of Satellite
+    name: MetaData/heightOfStation
+    source: variables/heightOfStation
+    units: m
+  - longName: Solar Zenith Angle
+    name: MetaData/solarZenithAngle
+    range:
+    - 0
+    - 180
+    source: variables/solarZenithAngle
+    units: degree
+  - longName: Solar Azimuth Angle
+    name: MetaData/solarAzimuthAngle
+    range:
+    - 0
+    - 360
+    source: variables/solarAzimuthAngle
+    units: degree
+  - longName: Sensor Zenith Angle
+    name: MetaData/sensorZenithAngle
+    range:
+    - 0
+    - 90
+    source: variables/sensorZenithAngle
+    units: degree
+  - longName: Sensor Azimuth Angle
+    name: MetaData/sensorAzimuthAngle
+    range:
+    - 0
+    - 360
+    source: variables/sensorAzimuthAngle
+    units: degree
+  - longName: Sensor View Angle
+    name: MetaData/sensorViewAngle
+    source: variables/sensorScanAngle
+    units: degree
+  - longName: Sensor Channel Number
+    name: MetaData/sensorChannelNumber
+    source: variables/sensorChannelNumber
+  - chunks:
+    - 1000
+    - 15
+    compressionLevel: 4
+    coordinates: longitude latitude Channel
+    longName: Brightness Temperature
+    name: ObsValue/brightnessTemperature
+    range:
+    - 100
+    - 500
+    source: variables/brightnessTemperature
+    units: K
+  - longName: Datetime
+    name: MetaData/dateTime
+    source: variables/timestamp
+    units: seconds since 1970-01-01T00:00:00Z
+  - longName: Latitude
+    name: MetaData/latitude
+    range:
+    - -90
+    - 90
+    source: variables/latitude
+    units: degree_north
+  - longName: Longitude
+    name: MetaData/longitude
+    source: variables/longitude
+    units: degree_east

dump/mapping/bufr2ioda_amsua.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+import bufr
+import netCDF4 as nc
+import os
+from pyioda.ioda.Engines.Bufr import Encoder
+from wxflow import Logger
+
+logger = Logger(os.path.basename(__file__), level='INFO', colored_log=True)
+
+YAML_NORMAL = False  # current as normal need remap for path2/1bama
+
+R1000 = 1000.0
+R1000000 = 1000000.0
+INVALID = R1000
+# Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE
+# Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520
+TSPACE = 2.7253
+
+nc_dir = '../../dump/aux'
+
+
+class ACCoeff:
+    def __init__(self, ac_dir, sat_id='n19'):
+        file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '.ACCoeff.nc')
+        nc_file = nc.Dataset(file_name)
+        self.n_fovs = len(nc_file.dimensions['n_FOVs'])
+        self.n_channels = len(nc_file.dimensions['n_Channels'])
+        self.a_earth = nc_file.variables['A_earth'][:]
+        self.a_platform = nc_file.variables['A_platform'][:]
+        self.a_space = nc_file.variables['A_space'][:]
+        self.a_ep = self.a_earth + self.a_platform
+        self.a_sp = self.a_space * TSPACE
+
+
+def remove_ant_corr(i, ac, ifov, t):
+    # AC:             Structure containing the antenna correction coefficients for the sensor of interest.
+    # iFOV:           The FOV index for a scanline of the sensor of interest.
+    # T:              On input, this argument contains the brightness
+
+    t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov]
+    t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID]
+    return t
+
+
+def apply_ant_corr(i, ac, ifov, t):
+    # t:              on input, this argument contains the antenna temperatures for the sensor channels.
+    t = (t - ac.a_sp[i, ifov]) / ac.a_ep[i, ifov]
+    t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID]
+    return t
+
+
+def get_description(yaml_path):
+    description = bufr.encoders.Description(yaml_path)
+    return description
+
+
+def apply_corr(sat_id, ta, ifov):
+    ac = ACCoeff(nc_dir, sat_id=sat_id)
+    if sat_id not in ['n15', 'n16']:
+        # Convert antenna temperature to brightness temperature
+        ifov = ifov.astype(int) - 1
+        for i in range(ta.shape[1]):
+            logger.debug(f'inside loop for allpy ta to tb: i = {i}')
+            x = ta[:, i]
+            if YAML_NORMAL:
+                x = apply_ant_corr(i, ac, ifov, x)
+            else:
+                x = remove_ant_corr(i, ac, ifov, x)
+            x[x >= R1000] = R1000000
+            ta[:, i] = x
+    return ta
+
+
+def re_map_variable(container):
+    # read_bufrtovs.f90
+    # antcorr_application.f90
+    # search the keyword “ta2tb” for details
+    sat_ids = container.all_sub_categories()
+    for sat_id in sat_ids:
+        logger.info(f'Converting for {sat_id[0]}, ...')
+        ta = container.get('variables/brightnessTemperature', sat_id)
+        if ta.shape[0]:
+            ifov = container.get('variables/fieldOfViewNumber', sat_id)
+            tb = apply_corr(sat_id[0], ta, ifov)
+            container.replace('variables/brightnessTemperature', tb, sat_id)
+
+
+def get_one_data(input_path, yaml_path, category):
+    cache = bufr.DataCache.has(input_path, yaml_path)
+    if cache:
+        logger.info(f'The cache existed get data container from it')
+        container = bufr.DataCache.get(input_path, yaml_path)
+    else:
+        # If cacache does not exist, get data into cache
+        # Get data info container first
+        logger.info(f'The cache is not existed')
+        container = bufr.Parser(input_path, yaml_path).parse()
+    return cache, container
+
+
+def mark_one_data(cache, input_path, yaml_path, category, container=None):
+    if cache:
+        logger.info(f'The cache existed get data container from it')
+        bufr.DataCache.mark_finished(input_path, yaml_path, [category])
+    else:
+        logger.info(f'add original container list into a cache = {container.list()}')
+        bufr.DataCache.add(input_path, yaml_path, container.all_sub_categories(), container)
+        bufr.DataCache.mark_finished(input_path, yaml_path, [category])
+
+
+def create_obs_group(input_path1, input_path2, yaml_1b, yaml_es, category):
+    logger.info(f'imput_path: {input_path1}, {input_path2}, and category: {category}')
+    logger.info(f'Entering function to create obs group for {category} with yaml path {yaml_es} and {yaml_1b}')
+    cache_1, container_1 = get_one_data(input_path1, yaml_es, category)
+    cache_2, container_2 = get_one_data(input_path2, yaml_1b, category)
+
+    re_map_variable(container_2)
+
+    container = container_1
+    container.append(container_2)
+
+    logger.info('Container append done')
+    data = Encoder(get_description(yaml_es)).encode(container)[(category,)]
+    mark_one_data(cache_1, input_path1, yaml_es, category, container=container_1)
+    mark_one_data(cache_2, input_path2, yaml_1b, category, container=container_2)
+    return data
+